From ab66c0861d32e266b2f197d69adcdbb177b4127f Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 2 Aug 2022 13:03:58 +0200 Subject: [PATCH 01/75] Add derivation of siextent --- .../cmor/tables/custom/CMOR_siextent.dat | 23 +++++++++ esmvalcore/preprocessor/_derive/siextent.py | 48 +++++++++++++++++++ .../preprocessor/_derive/test_siextent.py | 33 +++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 esmvalcore/cmor/tables/custom/CMOR_siextent.dat create mode 100644 esmvalcore/preprocessor/_derive/siextent.py create mode 100644 tests/unit/preprocessor/_derive/test_siextent.py diff --git a/esmvalcore/cmor/tables/custom/CMOR_siextent.dat b/esmvalcore/cmor/tables/custom/CMOR_siextent.dat new file mode 100644 index 0000000000..5dc87ae84a --- /dev/null +++ b/esmvalcore/cmor/tables/custom/CMOR_siextent.dat @@ -0,0 +1,23 @@ +SOURCE: CMIP5 +!============ +variable_entry: siextent +!============ +modeling_realm: seaIce +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: +units: 1 +cell_methods: area: mean where sea time: mean +cell_measures: area: areacello +long_name: Sea Ice Extent +comment: +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: longitude latitude time +type: real +valid_min: +valid_max: +!---------------------------------- +! diff --git a/esmvalcore/preprocessor/_derive/siextent.py b/esmvalcore/preprocessor/_derive/siextent.py new file mode 100644 index 0000000000..0b9695fcdf --- /dev/null +++ b/esmvalcore/preprocessor/_derive/siextent.py @@ -0,0 +1,48 @@ +"""Derivation of variable `sithick`.""" + +import dask.array as da + +from iris import Constraint + +from ._baseclass import DerivedVariableBase + + +class DerivedVariable(DerivedVariableBase): + """Derivation of variable `siextent`.""" + + @staticmethod + def required(project): + """Declare the variables needed for derivation.""" + required = [{ + 'short_name': 'sic', + }] + return required + + @staticmethod + def calculate(cubes): + """ + Compute sea ice extent. + + Returns an array of ones in every grid point where the sea ice area fraction + has values > 15 . + + Use in combination with the preprocessor `area_statistics(operator='sum')` to + weigh by the area and compute global or regional sea ice extent values. + + Arguments + --------- + cubes: cubelist containing volume and concentration components. + + Returns + ------- + Cube containing sea ice speed. + + """ + + siconc = cubes.extract_cube(Constraint(name='sea_ice_area_fraction')) + siconc.convert_units('1') + ones = da.ones_like(siconc) + siextent_data = da.ma.masked_where(siconc.lazy_data()>0.15, ones) + siextent = siconc.copy(siextent_data) + + return siextent diff --git a/tests/unit/preprocessor/_derive/test_siextent.py b/tests/unit/preprocessor/_derive/test_siextent.py new file mode 100644 index 0000000000..7d95f3c2bc --- /dev/null +++ b/tests/unit/preprocessor/_derive/test_siextent.py @@ -0,0 +1,33 @@ +"""Test derivation of `ohc`.""" +import cf_units +import iris +import numpy as np +import pytest + +import esmvalcore.preprocessor._derive.siextent as siextent + + +@pytest.fixture +def cubes(): + sic_name = 'sea_ice_area_fraction' + time_coord = iris.coords.DimCoord([0., 1., 2.], + standard_name='time') + sic_cube = iris.cube.Cube([[[15, 10], [10, 10]], + [[10, 10], [10, 10]], + [[10, 10], [10, 10]]], + units='%', + standard_name=sic_name, + var_name='siconc', + dim_coords_and_dims=[(time_coord, 0)]) + return iris.cube.CubeList([sic_cube]) + + +def test_siextent_calculation(cubes): + derived_var = siextent.DerivedVariable() + out_cube = derived_var.calculate(cubes) + assert out_cube.units == cf_units.Unit('1') + out_data = out_cube.data + expected = np.ma.ones_like(cubes[0].data) + expected.mask = True + expected[0][0][0] = 1. + np.testing.assert_array_equal(out_data, expected) From 079767bd05708ec49bb8365bd8147b568b8f0c3b Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 8 Aug 2022 14:26:44 +0200 Subject: [PATCH 02/75] Set proper units --- esmvalcore/preprocessor/_derive/siextent.py | 17 +++++++++-------- .../unit/preprocessor/_derive/test_siextent.py | 4 ++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/esmvalcore/preprocessor/_derive/siextent.py b/esmvalcore/preprocessor/_derive/siextent.py index 0b9695fcdf..61e0ad128f 100644 --- a/esmvalcore/preprocessor/_derive/siextent.py +++ b/esmvalcore/preprocessor/_derive/siextent.py @@ -23,26 +23,27 @@ def calculate(cubes): """ Compute sea ice extent. - Returns an array of ones in every grid point where the sea ice area fraction - has values > 15 . + Returns an array of ones in every grid point where + the sea ice area fraction has values > 15 . - Use in combination with the preprocessor `area_statistics(operator='sum')` to - weigh by the area and compute global or regional sea ice extent values. + Use in combination with the preprocessor + `area_statistics(operator='sum')` to weigh by the area and + compute global or regional sea ice extent values. Arguments --------- - cubes: cubelist containing volume and concentration components. + cubes: cubelist containing sea ice area fraction. Returns ------- - Cube containing sea ice speed. + Cube containing sea ice extent. """ siconc = cubes.extract_cube(Constraint(name='sea_ice_area_fraction')) - siconc.convert_units('1') ones = da.ones_like(siconc) - siextent_data = da.ma.masked_where(siconc.lazy_data()>0.15, ones) + siextent_data = da.ma.masked_where(siconc.lazy_data()<15., ones) siextent = siconc.copy(siextent_data) + siextent.units = 'm2' return siextent diff --git a/tests/unit/preprocessor/_derive/test_siextent.py b/tests/unit/preprocessor/_derive/test_siextent.py index 7d95f3c2bc..b618b1ead1 100644 --- a/tests/unit/preprocessor/_derive/test_siextent.py +++ b/tests/unit/preprocessor/_derive/test_siextent.py @@ -12,7 +12,7 @@ def cubes(): sic_name = 'sea_ice_area_fraction' time_coord = iris.coords.DimCoord([0., 1., 2.], standard_name='time') - sic_cube = iris.cube.Cube([[[15, 10], [10, 10]], + sic_cube = iris.cube.Cube([[[20, 10], [10, 10]], [[10, 10], [10, 10]], [[10, 10], [10, 10]]], units='%', @@ -25,7 +25,7 @@ def cubes(): def test_siextent_calculation(cubes): derived_var = siextent.DerivedVariable() out_cube = derived_var.calculate(cubes) - assert out_cube.units == cf_units.Unit('1') + assert out_cube.units == cf_units.Unit('m2') out_data = out_cube.data expected = np.ma.ones_like(cubes[0].data) expected.mask = True From 28340289df217b98aa15083275451526e0b93a22 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 8 Aug 2022 14:38:59 +0200 Subject: [PATCH 03/75] Fix tests --- esmvalcore/preprocessor/_derive/siextent.py | 6 +++--- tests/unit/preprocessor/_derive/test_siextent.py | 15 ++++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/esmvalcore/preprocessor/_derive/siextent.py b/esmvalcore/preprocessor/_derive/siextent.py index 61e0ad128f..d12b99328e 100644 --- a/esmvalcore/preprocessor/_derive/siextent.py +++ b/esmvalcore/preprocessor/_derive/siextent.py @@ -23,11 +23,11 @@ def calculate(cubes): """ Compute sea ice extent. - Returns an array of ones in every grid point where + Returns an array of ones in every grid point where the sea ice area fraction has values > 15 . - Use in combination with the preprocessor - `area_statistics(operator='sum')` to weigh by the area and + Use in combination with the preprocessor + `area_statistics(operator='sum')` to weigh by the area and compute global or regional sea ice extent values. Arguments diff --git a/tests/unit/preprocessor/_derive/test_siextent.py b/tests/unit/preprocessor/_derive/test_siextent.py index b618b1ead1..617917c15c 100644 --- a/tests/unit/preprocessor/_derive/test_siextent.py +++ b/tests/unit/preprocessor/_derive/test_siextent.py @@ -13,12 +13,12 @@ def cubes(): time_coord = iris.coords.DimCoord([0., 1., 2.], standard_name='time') sic_cube = iris.cube.Cube([[[20, 10], [10, 10]], - [[10, 10], [10, 10]], - [[10, 10], [10, 10]]], - units='%', - standard_name=sic_name, - var_name='siconc', - dim_coords_and_dims=[(time_coord, 0)]) + [[10, 10], [10, 10]], + [[10, 10], [10, 10]]], + units='%', + standard_name=sic_name, + var_name='siconc', + dim_coords_and_dims=[(time_coord, 0)]) return iris.cube.CubeList([sic_cube]) @@ -30,4 +30,5 @@ def test_siextent_calculation(cubes): expected = np.ma.ones_like(cubes[0].data) expected.mask = True expected[0][0][0] = 1. - np.testing.assert_array_equal(out_data, expected) + np.testing.assert_array_equal(out_data.mask, expected.mask) + np.testing.assert_array_equal(out_data[0][0][0], expected[0][0][0]) From 119cec1755bfdf0230e3442340c25a2ff662a20e Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 8 Aug 2022 14:45:17 +0200 Subject: [PATCH 04/75] Fix flake --- esmvalcore/preprocessor/_derive/siextent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_derive/siextent.py b/esmvalcore/preprocessor/_derive/siextent.py index d12b99328e..e9418813df 100644 --- a/esmvalcore/preprocessor/_derive/siextent.py +++ b/esmvalcore/preprocessor/_derive/siextent.py @@ -39,10 +39,10 @@ def calculate(cubes): Cube containing sea ice extent. """ - + siconc = cubes.extract_cube(Constraint(name='sea_ice_area_fraction')) ones = da.ones_like(siconc) - siextent_data = da.ma.masked_where(siconc.lazy_data()<15., ones) + siextent_data = da.ma.masked_where(siconc.lazy_data() < 15., ones) siextent = siconc.copy(siextent_data) siextent.units = 'm2' From 6d7b96306e5422dcb7dd7fac3605aee69e537b7d Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 9 Aug 2022 14:37:58 +0200 Subject: [PATCH 05/75] Attempt to improve test coverage --- tests/unit/preprocessor/_derive/test_siextent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/preprocessor/_derive/test_siextent.py b/tests/unit/preprocessor/_derive/test_siextent.py index 617917c15c..7da57242b4 100644 --- a/tests/unit/preprocessor/_derive/test_siextent.py +++ b/tests/unit/preprocessor/_derive/test_siextent.py @@ -17,7 +17,7 @@ def cubes(): [[10, 10], [10, 10]]], units='%', standard_name=sic_name, - var_name='siconc', + var_name='sic', dim_coords_and_dims=[(time_coord, 0)]) return iris.cube.CubeList([sic_cube]) From 7690b59c48707b47d90482473f809b188725790f Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 15 Sep 2022 10:56:26 +0200 Subject: [PATCH 06/75] Improve test coverage --- esmvalcore/preprocessor/_derive/siextent.py | 8 ++++---- tests/unit/preprocessor/_derive/test_siextent.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/esmvalcore/preprocessor/_derive/siextent.py b/esmvalcore/preprocessor/_derive/siextent.py index e9418813df..dae973b136 100644 --- a/esmvalcore/preprocessor/_derive/siextent.py +++ b/esmvalcore/preprocessor/_derive/siextent.py @@ -40,10 +40,10 @@ def calculate(cubes): """ - siconc = cubes.extract_cube(Constraint(name='sea_ice_area_fraction')) - ones = da.ones_like(siconc) - siextent_data = da.ma.masked_where(siconc.lazy_data() < 15., ones) - siextent = siconc.copy(siextent_data) + sic = cubes.extract_cube(Constraint(name='sea_ice_area_fraction')) + ones = da.ones_like(sic) + siextent_data = da.ma.masked_where(sic.lazy_data() < 15., ones) + siextent = sic.copy(siextent_data) siextent.units = 'm2' return siextent diff --git a/tests/unit/preprocessor/_derive/test_siextent.py b/tests/unit/preprocessor/_derive/test_siextent.py index 7da57242b4..cb58ad2599 100644 --- a/tests/unit/preprocessor/_derive/test_siextent.py +++ b/tests/unit/preprocessor/_derive/test_siextent.py @@ -23,6 +23,7 @@ def cubes(): def test_siextent_calculation(cubes): + """Test function ``calculate``.""" derived_var = siextent.DerivedVariable() out_cube = derived_var.calculate(cubes) assert out_cube.units == cf_units.Unit('m2') @@ -32,3 +33,12 @@ def test_siextent_calculation(cubes): expected[0][0][0] = 1. np.testing.assert_array_equal(out_data.mask, expected.mask) np.testing.assert_array_equal(out_data[0][0][0], expected[0][0][0]) + + +def test_siextent_required(): + """Test function ``required``.""" + derived_var = siextent.DerivedVariable() + output = derived_var.required(None) + assert output == [ + {'short_name': 'sic'} + ] From b8b6248f223c79d4165cfe42377b4abba57db996 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 15 Sep 2022 13:00:38 +0200 Subject: [PATCH 07/75] Change units in table --- esmvalcore/cmor/tables/custom/CMOR_siextent.dat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/cmor/tables/custom/CMOR_siextent.dat b/esmvalcore/cmor/tables/custom/CMOR_siextent.dat index 5dc87ae84a..736744595f 100644 --- a/esmvalcore/cmor/tables/custom/CMOR_siextent.dat +++ b/esmvalcore/cmor/tables/custom/CMOR_siextent.dat @@ -7,7 +7,7 @@ modeling_realm: seaIce ! Variable attributes: !---------------------------------- standard_name: -units: 1 +units: m2 cell_methods: area: mean where sea time: mean cell_measures: area: areacello long_name: Sea Ice Extent From 08384666283c4263ee0bd27cd87c175b314e8e72 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 3 Oct 2022 09:31:47 +0200 Subject: [PATCH 08/75] Consider using siconca --- .../cmor/tables/custom/CMOR_siextent.dat | 2 +- esmvalcore/preprocessor/_derive/siextent.py | 37 +++++++++++++------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/esmvalcore/cmor/tables/custom/CMOR_siextent.dat b/esmvalcore/cmor/tables/custom/CMOR_siextent.dat index 5dc87ae84a..736744595f 100644 --- a/esmvalcore/cmor/tables/custom/CMOR_siextent.dat +++ b/esmvalcore/cmor/tables/custom/CMOR_siextent.dat @@ -7,7 +7,7 @@ modeling_realm: seaIce ! Variable attributes: !---------------------------------- standard_name: -units: 1 +units: m2 cell_methods: area: mean where sea time: mean cell_measures: area: areacello long_name: Sea Ice Extent diff --git a/esmvalcore/preprocessor/_derive/siextent.py b/esmvalcore/preprocessor/_derive/siextent.py index e9418813df..d59335f8dd 100644 --- a/esmvalcore/preprocessor/_derive/siextent.py +++ b/esmvalcore/preprocessor/_derive/siextent.py @@ -1,11 +1,14 @@ """Derivation of variable `sithick`.""" +import logging import dask.array as da - +import iris from iris import Constraint from ._baseclass import DerivedVariableBase +logger = logging.getLogger(__name__) + class DerivedVariable(DerivedVariableBase): """Derivation of variable `siextent`.""" @@ -13,15 +16,20 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{ - 'short_name': 'sic', - }] + required = [ + { + 'short_name': 'sic', + 'optional': 'true' + }, + { + 'short_name': 'siconca', + 'optional': 'true' + }] return required @staticmethod def calculate(cubes): - """ - Compute sea ice extent. + """Compute sea ice extent. Returns an array of ones in every grid point where the sea ice area fraction has values > 15 . @@ -37,13 +45,20 @@ def calculate(cubes): Returns ------- Cube containing sea ice extent. - """ + try: + sic = cubes.extract_cube(Constraint(name='sic')) + except iris.exceptions.ConstraintMismatchError: + try: + sic = cubes.extract_cube(Constraint(name='siconca')) + except iris.exceptions.ConstraintMismatchError: + logger.error( + 'Derivation of siextent failed due to missing variables ' + 'sic and siconca.') - siconc = cubes.extract_cube(Constraint(name='sea_ice_area_fraction')) - ones = da.ones_like(siconc) - siextent_data = da.ma.masked_where(siconc.lazy_data() < 15., ones) - siextent = siconc.copy(siextent_data) + ones = da.ones_like(sic) + siextent_data = da.ma.masked_where(sic.lazy_data() < 15., ones) + siextent = sic.copy(siextent_data) siextent.units = 'm2' return siextent From 09a6da76079791b02074fbd668775d52d19f7f86 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 3 Oct 2022 10:02:06 +0200 Subject: [PATCH 09/75] Fix tests --- .../preprocessor/_derive/test_siextent.py | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/tests/unit/preprocessor/_derive/test_siextent.py b/tests/unit/preprocessor/_derive/test_siextent.py index cb58ad2599..3734216b43 100644 --- a/tests/unit/preprocessor/_derive/test_siextent.py +++ b/tests/unit/preprocessor/_derive/test_siextent.py @@ -8,7 +8,7 @@ @pytest.fixture -def cubes(): +def cubes_sic(): sic_name = 'sea_ice_area_fraction' time_coord = iris.coords.DimCoord([0., 1., 2.], standard_name='time') @@ -21,14 +21,41 @@ def cubes(): dim_coords_and_dims=[(time_coord, 0)]) return iris.cube.CubeList([sic_cube]) +@pytest.fixture +def cubes_siconca(): + sic_name = 'sea_ice_area_fraction' + time_coord = iris.coords.DimCoord([0., 1., 2.], + standard_name='time') + sic_cube = iris.cube.Cube([[[20, 10], [10, 10]], + [[10, 10], [10, 10]], + [[10, 10], [10, 10]]], + units='%', + standard_name=sic_name, + var_name='siconca', + dim_coords_and_dims=[(time_coord, 0)]) + return iris.cube.CubeList([sic_cube]) + + +def test_siextent_calculation_sic(cubes_sic): + """Test function ``calculate`` when sic is available.""" + derived_var = siextent.DerivedVariable() + out_cube = derived_var.calculate(cubes_sic) + assert out_cube.units == cf_units.Unit('m2') + out_data = out_cube.data + expected = np.ma.ones_like(cubes_sic[0].data) + expected.mask = True + expected[0][0][0] = 1. + np.testing.assert_array_equal(out_data.mask, expected.mask) + np.testing.assert_array_equal(out_data[0][0][0], expected[0][0][0]) + -def test_siextent_calculation(cubes): - """Test function ``calculate``.""" +def test_siextent_calculation_siconca(cubes_siconca): + """Test function ``calculate`` when siconca is available.""" derived_var = siextent.DerivedVariable() - out_cube = derived_var.calculate(cubes) + out_cube = derived_var.calculate(cubes_siconca) assert out_cube.units == cf_units.Unit('m2') out_data = out_cube.data - expected = np.ma.ones_like(cubes[0].data) + expected = np.ma.ones_like(cubes_siconca[0].data) expected.mask = True expected[0][0][0] = 1. np.testing.assert_array_equal(out_data.mask, expected.mask) @@ -40,5 +67,6 @@ def test_siextent_required(): derived_var = siextent.DerivedVariable() output = derived_var.required(None) assert output == [ - {'short_name': 'sic'} + {'short_name': 'sic', 'optional': 'true'}, + {'short_name': 'siconca', 'optional': 'true'} ] From 022aec243d273c09c04e6c334c3c4a7814dca75b Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 3 Oct 2022 10:10:05 +0200 Subject: [PATCH 10/75] Fix tests --- .../preprocessor/_derive/test_siextent.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/unit/preprocessor/_derive/test_siextent.py b/tests/unit/preprocessor/_derive/test_siextent.py index 3734216b43..1561ddc79a 100644 --- a/tests/unit/preprocessor/_derive/test_siextent.py +++ b/tests/unit/preprocessor/_derive/test_siextent.py @@ -21,6 +21,7 @@ def cubes_sic(): dim_coords_and_dims=[(time_coord, 0)]) return iris.cube.CubeList([sic_cube]) + @pytest.fixture def cubes_siconca(): sic_name = 'sea_ice_area_fraction' @@ -36,6 +37,27 @@ def cubes_siconca(): return iris.cube.CubeList([sic_cube]) +@pytest.fixture +def cubes(): + sic_name = 'sea_ice_area_fraction' + time_coord = iris.coords.DimCoord([0., 1., 2.], + standard_name='time') + sic_cube = iris.cube.Cube([[[20, 10], [10, 10]], + [[10, 10], [10, 10]], + [[10, 10], [10, 10]]], + units='%', + standard_name=sic_name, + var_name='sic', + dim_coords_and_dims=[(time_coord, 0)]) + siconca_cube = iris.cube.Cube([[[20, 10], [10, 10]], + [[10, 10], [10, 10]], + [[10, 10], [10, 10]]], + units='%', + standard_name=sic_name, + var_name='siconca', + dim_coords_and_dims=[(time_coord, 0)]) + return iris.cube.CubeList([sic_cube, siconca_cube]) + def test_siextent_calculation_sic(cubes_sic): """Test function ``calculate`` when sic is available.""" derived_var = siextent.DerivedVariable() @@ -62,6 +84,19 @@ def test_siextent_calculation_siconca(cubes_siconca): np.testing.assert_array_equal(out_data[0][0][0], expected[0][0][0]) +def test_siextent_calculation(cubes): + """Test function ``calculate`` when sic and siconca are available.""" + derived_var = siextent.DerivedVariable() + out_cube = derived_var.calculate(cubes) + assert out_cube.units == cf_units.Unit('m2') + out_data = out_cube.data + expected = np.ma.ones_like(cubes[0].data) + expected.mask = True + expected[0][0][0] = 1. + np.testing.assert_array_equal(out_data.mask, expected.mask) + np.testing.assert_array_equal(out_data[0][0][0], expected[0][0][0]) + + def test_siextent_required(): """Test function ``required``.""" derived_var = siextent.DerivedVariable() From 85c1f0c8273bc510dbde060eb6ecfe876064acd4 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 3 Oct 2022 10:13:01 +0200 Subject: [PATCH 11/75] Add missing line --- tests/unit/preprocessor/_derive/test_siextent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/preprocessor/_derive/test_siextent.py b/tests/unit/preprocessor/_derive/test_siextent.py index 1561ddc79a..0b21c4b216 100644 --- a/tests/unit/preprocessor/_derive/test_siextent.py +++ b/tests/unit/preprocessor/_derive/test_siextent.py @@ -58,6 +58,7 @@ def cubes(): dim_coords_and_dims=[(time_coord, 0)]) return iris.cube.CubeList([sic_cube, siconca_cube]) + def test_siextent_calculation_sic(cubes_sic): """Test function ``calculate`` when sic is available.""" derived_var = siextent.DerivedVariable() From 44ed216867d4e13a0716aabac7a2020b12203b76 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 17 Oct 2022 15:24:26 +0200 Subject: [PATCH 12/75] Add cordex fixes --- esmvalcore/cmor/_fixes/cordex/__init__ .py | 1 + .../cordex/cnrm_cerfacs_cnrm_cm5/__init__.py | 1 + .../cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py | 58 ++++++++++++++ .../mohc_hadrem3_ga7_05.py | 5 ++ .../_fixes/cordex/ichec_ec_earth/__init__.py | 1 + .../ichec_ec_earth/clmcom_cclm4_8_17.py | 45 +++++++++++ .../cordex/ichec_ec_earth/dmi_hirham5.py | 10 +++ .../cordex/ichec_ec_earth/gerics_remo2015.py | 18 +++++ .../cordex/ichec_ec_earth/knmi_racmo22e.py | 4 + .../ichec_ec_earth/mohc_hadrem3_ga7_05.py | 11 +++ .../_fixes/cordex/ichec_ec_earth/smhi_rca4.py | 5 ++ .../cordex/miroc_miroc5/clmcom_cclm4_8_17.py | 45 +++++++++++ .../cordex/miroc_miroc5/gerics_remo2015.py | 4 + .../cordex/miroc_miroc5/uhoh_wrf361h.py | 77 +++++++++++++++++++ .../_fixes/cordex/mohc_hadgem2_es/__init__.py | 1 + .../cordex/mohc_hadgem2_es/dmi_hirham5.py | 28 +++++++ .../cordex/mohc_hadgem2_es/gerics_remo2015.py | 17 ++++ .../cordex/mohc_hadgem2_es/ictp_regcm4_6.py | 6 ++ .../mohc_hadgem2_es/mohc_hadrem3_ga7_05.py | 33 ++++++++ .../cordex/mohc_hadgem2_es/smhi_rca4.py | 5 ++ .../cordex/mpi_m_mpi_esm_lr/__init__.py | 1 + .../cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py | 13 ++++ .../cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py | 4 + .../mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py | 11 +++ .../cordex/mpi_m_mpi_esm_lr/uhoh_wrf361h.py | 2 + .../_fixes/cordex/ncc_noresm1_m/__init__.py | 1 + .../cordex/ncc_noresm1_m/gerics_remo2015.py | 32 ++++++++ .../cordex/ncc_noresm1_m/knmi_racmo22e.py | 4 + .../ncc_noresm1_m/mohc_hadrem3_ga7_05.py | 11 +++ .../_fixes/cordex/ncc_noresm1_m/smhi_rca4.py | 5 ++ 30 files changed, 459 insertions(+) create mode 100644 esmvalcore/cmor/_fixes/cordex/__init__ .py create mode 100644 esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py create mode 100644 esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py create mode 100644 esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py create mode 100644 esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py create mode 100644 esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/ictp_regcm4_6.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py create mode 100644 esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/uhoh_wrf361h.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py create mode 100644 esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py diff --git a/esmvalcore/cmor/_fixes/cordex/__init__ .py b/esmvalcore/cmor/_fixes/cordex/__init__ .py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/__init__ .py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/__init__.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py new file mode 100644 index 0000000000..e9018ca815 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py @@ -0,0 +1,58 @@ +# height 2m a differents altures +from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor._fixes.shared import add_scalar_height_coord + +import numpy as np + +class Tas(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """ + Add height (2m) coordinate. + + Fix also done for prw. + Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + add_scalar_height_coord(cube) + if cube.coord('height').points != 2.: + cube.coord('height').points = np.ma.array([2.0]) + cube.coord('time').long_name = 'time' + + return cubes + +class Pr(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """ + Add height (2m) coordinate. + + Fix also done for prw. + Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + cube.coord('time').long_name = 'time' + + return cubes \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py new file mode 100644 index 0000000000..b4e109c907 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py @@ -0,0 +1,5 @@ +from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix + +Tas = BaseFix + +Pr = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/__init__.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py new file mode 100644 index 0000000000..92d1396298 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py @@ -0,0 +1,45 @@ +# height 2m a differents altures +from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor._fixes.shared import add_scalar_height_coord + +from cf_units import Unit +import iris +import numpy as np + +class AllVars(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """ + Add height (2m) coordinate. + + Fix also done for prw. + Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coord('time').units.calendar == 'gregorian': + cube.coord('time').units = Unit( + cube.coord('time').units.cftime_unit, + calendar='proleptic_gregorian' + ) + for coord in cube.coords(): + if coord.dtype in ['>f8', '>f4']: + coord.points = coord.core_points().astype( + np.float64, casting='same_kind') + if coord.bounds is not None: + coord.bounds = coord.core_bounds().astype( + np.float64, casting='same_kind') + # further issues with dtype, may be due because historical data lat/lon does not have bounds whereas scenario data has them + + + return cubes \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py new file mode 100644 index 0000000000..5905526fe0 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py @@ -0,0 +1,10 @@ + +from esmvalcore.cmor.fix import Fix + +#rcp85 data does not have grid_lat and grid_lon defined +# Traceback (most recent call last): +# File "", line 1, in +# File "/home/b/b381943/ESMValCore/esmvalcore/preprocessor/_io.py", line 220, in _get_concatenation_error +# raise ValueError(f'Can not concatenate cubes: {msg}') +# ValueError: Can not concatenate cubes: failed to concatenate into a single cube. +# Dimension coordinates differ: grid_latitude, grid_longitude, time != time \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py new file mode 100644 index 0000000000..51dc18a696 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py @@ -0,0 +1,18 @@ +# height 2m a differents altures +from esmvalcore.cmor.fix import Fix +from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix + + +Pr = BaseFix + +# for historical + rcp45 in Pr Amon, longitudes are shifted + +# cubes[0].coord('longitude').points - cubes[1].coord('longitude').points +# masked_array( +# data=[[-360., -360., -360., ..., 0., 0., 0.], +# [-360., -360., -360., ..., 0., 0., 0.], +# [-360., -360., -360., ..., 0., 0., 0.], +# ..., +# [-360., -360., -360., ..., 0., 0., 0.], +# [-360., -360., -360., ..., 0., 0., 0.], +# [-360., -360., -360., ..., 0., 0., 0.]], \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py new file mode 100644 index 0000000000..abff52e381 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py @@ -0,0 +1,4 @@ +from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix + + +Pr = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py new file mode 100644 index 0000000000..97abb80f26 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py @@ -0,0 +1,11 @@ + +from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix + +import numpy as np + +Tas = BaseFix + +Pr = BaseFix + + + diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py new file mode 100644 index 0000000000..860a814ba6 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py @@ -0,0 +1,5 @@ +from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix + + +Pr = BaseFix +Tas = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py new file mode 100644 index 0000000000..299ed8e561 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py @@ -0,0 +1,45 @@ +# height 2m a differents altures +from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor._fixes.shared import add_scalar_height_coord + +from cf_units import Unit +import iris +import numpy as np + +class AllVars(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """ + Add height (2m) coordinate. + + Fix also done for prw. + Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coord('time').units.calendar == 'gregorian': + cube.coord('time').units = Unit( + 'days since 1850-1-1 00:00:00', + calendar='proleptic_gregorian' + ) + for coord in cube.coords(): + if coord.dtype in ['>f8', '>f4']: + coord.points = coord.core_points().astype( + np.float64, casting='same_kind') + if coord.bounds is not None: + coord.bounds = coord.core_bounds().astype( + np.float64, casting='same_kind') + # further issues with dtype, may be due because historical data lat/lon does not have bounds whereas scenario data has them + + + return cubes \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py new file mode 100644 index 0000000000..a29514edc8 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py @@ -0,0 +1,4 @@ +# height 2m a differents altures +from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix + +Pr = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py new file mode 100644 index 0000000000..9836d2ea62 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py @@ -0,0 +1,77 @@ +from esmvalcore.cmor.fix import Fix + +import iris + +class Tas(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """ + Add height (2m) coordinate. + + Fix also done for prw. + Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + squeezed = [] + for cube in cubes: + height = cube.coord('height') + if isinstance(height, iris.coords.DimCoord): + squeezed.append(iris.util.squeeze(cube)) + + if squeezed: + return squeezed + + return cubes + +# for tas / pr rcp85, lat and lon do not exist + + +# esmvalcore.cmor.check.CMORCheckError: There were errors in variable pr: +# longitude: does not exist +# latitude: does not exist +# pr: does not match coordinate rank +# in cube: +# precipitation_flux / (kg m-2 s-1) (time: 60; -- : 412; -- : 424) +# Dimension coordinates: +# time x - - +# Cell methods: +# mean time +# Attributes: +# CDO 'Climate Data Operators version 1.9.4 (http://mpimet.mpg.de/cdo)' +# CORDEX_domain 'EUR-11' +# Conventions 'CF-1.4' +# NCO '20180925' +# c3s_disclaimer 'This data has been produced in the context of PRINCIPLE/CORDEX4CDS project... +# contact 'viktoria.mohr@uni-hohenheim.de' +# driving_experiment 'MIROC-MIROC5, rcp85, r1i1p1' +# driving_experiment_name 'rcp85' +# driving_model_ensemble_member 'r1i1p1' +# driving_model_id 'MIROC-MIROC5' +# experiment 'rcp85' +# experiment_id 'rcp85' +# frequency 'mon' +# institute_id 'UHOH' +# institute_run_id 'hoh' +# institution 'Institute of Physics an Meteorology University of Hohenheim (UHOH), G... +# model_id 'UHOH-WRF361H' +# nco_openmp_thread_number 1 +# product 'output' +# project_id 'CORDEX' +# rcm_version_id 'v1' +# references 'https://www120.uni-hohenheim.de' +# source_file '/work/ik1017/C3SCORDEX/data/c3s-cordex/output/EUR-11/UHOH/MIROC-MIROC... +# loaded from file /work/ik1017/C3SCORDEX/data/c3s-cordex/output/EUR-11/UHOH/MIROC-MIROC5/rcp85/r1i1p1/UHOH-WRF361H/v1/mon/pr/v20180717/pr_EUR-11_MIROC-MIROC5_rcp85_r1i1p1_UHOH-WRF361H_v1_mon_200601-201012.nc +# 2022-10-11 10:04:10,988 UTC [3437717] INFO +# If you have a question or need help, please start a new discussion on https://github.com/ESMValGroup/ESMValTool/discussions +# If you suspect this is a bug, please open an issue on https://github.com/ESMValGroup/ESMValTool/issues +# To make it easier to find out what the problem is, please consider attaching the files run/recipe_*.yml and run/main_log_debug.txt from the output directory. diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/__init__.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py new file mode 100644 index 0000000000..a4a95eb8f2 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py @@ -0,0 +1,28 @@ + +from esmvalcore.cmor.fix import Fix + +class Pr(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """ + Add height (2m) coordinate. + + Fix also done for prw. + Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + cube.coord('latitude').attributes = {} + cube.coord('longitude').attributes = {} + + return cubes \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py new file mode 100644 index 0000000000..dfd992b2ab --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py @@ -0,0 +1,17 @@ +from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix + + +Pr = BaseFix +Tas = BaseFix + +# for historical + rcp45 in Pr Amon, longitudes are shifted + +# cubes[0].coord('longitude').points - cubes[1].coord('longitude').points +# masked_array( +# data=[[-360., -360., -360., ..., 0., 0., 0.], +# [-360., -360., -360., ..., 0., 0., 0.], +# [-360., -360., -360., ..., 0., 0., 0.], +# ..., +# [-360., -360., -360., ..., 0., 0., 0.], +# [-360., -360., -360., ..., 0., 0., 0.], +# [-360., -360., -360., ..., 0., 0., 0.]], \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/ictp_regcm4_6.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/ictp_regcm4_6.py new file mode 100644 index 0000000000..3a732e783c --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/ictp_regcm4_6.py @@ -0,0 +1,6 @@ +# 3hr data between historical and scenario cannot be concatenated because historical +# finishes at +# Cell(point=cftime.DatetimeGregorian(2005, 12, 31, 21, 0, 0, 0, has_year_zero=False), bound=None) +# and scenario finishes at +# Cell(point=cftime.DatetimeGregorian(2006, 1, 1, 3, 0, 0, 0, has_year_zero=False), bound=None) +# which is 6h apart instead of 3h diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py new file mode 100644 index 0000000000..c100d08b15 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py @@ -0,0 +1,33 @@ + +from esmvalcore.cmor.fix import Fix + +import numpy as np + +class Tas(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """ + Add height (2m) coordinate. + + Fix also done for prw. + Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + cube.coord('latitude').var_name = 'lat' + cube.coord('longitude').var_name = 'lon' + cube.coord('time').long_name = 'time' + + return cubes + +Pr = Tas \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py new file mode 100644 index 0000000000..860a814ba6 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py @@ -0,0 +1,5 @@ +from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix + + +Pr = BaseFix +Tas = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/__init__.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py new file mode 100644 index 0000000000..27825f8ba0 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py @@ -0,0 +1,13 @@ +# 3hr data between historical and scenario cannot be concatenated because historical +# finishes at +# Cell(point=cftime.DatetimeGregorian(2005, 12, 31, 21, 0, 0, 0, has_year_zero=False), bound=None) +# and scenario finishes at +# Cell(point=cftime.DatetimeGregorian(2006, 1, 1, 3, 0, 0, 0, has_year_zero=False), bound=None) +# which is 6h apart instead of 3h + + +from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix + + +Pr = BaseFix +Tas = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py new file mode 100644 index 0000000000..abff52e381 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py @@ -0,0 +1,4 @@ +from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix + + +Pr = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py new file mode 100644 index 0000000000..97abb80f26 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py @@ -0,0 +1,11 @@ + +from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix + +import numpy as np + +Tas = BaseFix + +Pr = BaseFix + + + diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/uhoh_wrf361h.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/uhoh_wrf361h.py new file mode 100644 index 0000000000..d5800337a7 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/uhoh_wrf361h.py @@ -0,0 +1,2 @@ +# Rounding errors between historical and scenario exp in grid_latitude and grid_longitude + diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/__init__.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py new file mode 100644 index 0000000000..e68f39d014 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py @@ -0,0 +1,32 @@ +# height 2m a differents altures +from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor._fixes.shared import add_scalar_height_coord + +from cf_units import Unit +import iris +import numpy as np + +class Pr(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """ + Add height (2m) coordinate. + + Fix also done for prw. + Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + cube.coord('time').long_name = 'time' + + return cubes \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py new file mode 100644 index 0000000000..e9b59eab12 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py @@ -0,0 +1,4 @@ +from .gerics_remo2015 import Pr as BaseFix + + +Pr = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py new file mode 100644 index 0000000000..97abb80f26 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py @@ -0,0 +1,11 @@ + +from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix + +import numpy as np + +Tas = BaseFix + +Pr = BaseFix + + + diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py new file mode 100644 index 0000000000..4f11297652 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py @@ -0,0 +1,5 @@ +from .gerics_remo2015 import Pr as BaseFix + + +Pr = BaseFix +Tas = BaseFix \ No newline at end of file From 7e8192270f1bb2eb0b4ffca1fc9b3d2fabb386c6 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 17 Oct 2022 15:29:10 +0200 Subject: [PATCH 13/75] Load cordex fixes module --- esmvalcore/cmor/_fixes/fix.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index be94c4a774..7e473c36ba 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -158,8 +158,13 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): fixes = [] try: - fixes_module = importlib.import_module( - 'esmvalcore.cmor._fixes.{0}.{1}'.format(project, dataset)) + if project == 'cordex': + driver = extra_facets['driver'].replace('-', '_').lower() + fixes_module = importlib.import_module( + f'esmvalcore.cmor._fixes.{project}.{driver}.{dataset}') + else: + fixes_module = importlib.import_module( + 'esmvalcore.cmor._fixes.{0}.{1}'.format(project, dataset)) classes = inspect.getmembers(fixes_module, inspect.isclass) classes = dict((name.lower(), value) for name, value in classes) From f086c670b96853178b4bf6154526b171165c1b59 Mon Sep 17 00:00:00 2001 From: Pep Cos Date: Tue, 18 Oct 2022 16:54:16 +0200 Subject: [PATCH 14/75] clean fixes strings and format --- .../cordex/ichec_ec_earth/gerics_remo2015.py | 17 ++--------------- .../cordex/ichec_ec_earth/knmi_racmo22e.py | 1 + .../ichec_ec_earth/mohc_hadrem3_ga7_05.py | 8 ++------ .../_fixes/cordex/ichec_ec_earth/smhi_rca4.py | 2 ++ .../cordex/mohc_hadgem2_es/dmi_hirham5.py | 9 +++------ .../cordex/mohc_hadgem2_es/gerics_remo2015.py | 14 ++------------ .../mohc_hadgem2_es/mohc_hadrem3_ga7_05.py | 9 +++------ .../_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py | 2 ++ .../cordex/ncc_noresm1_m/gerics_remo2015.py | 13 +++---------- .../cordex/ncc_noresm1_m/knmi_racmo22e.py | 1 + .../cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py | 8 ++------ .../_fixes/cordex/ncc_noresm1_m/smhi_rca4.py | 2 ++ 12 files changed, 25 insertions(+), 61 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py index 51dc18a696..58bc498941 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py @@ -1,18 +1,5 @@ -# height 2m a differents altures -from esmvalcore.cmor.fix import Fix +"""Fixes for rcm GERICS-REMO2015 driven by ICHEC-EC-EARTH.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix -Pr = BaseFix - -# for historical + rcp45 in Pr Amon, longitudes are shifted - -# cubes[0].coord('longitude').points - cubes[1].coord('longitude').points -# masked_array( -# data=[[-360., -360., -360., ..., 0., 0., 0.], -# [-360., -360., -360., ..., 0., 0., 0.], -# [-360., -360., -360., ..., 0., 0., 0.], -# ..., -# [-360., -360., -360., ..., 0., 0., 0.], -# [-360., -360., -360., ..., 0., 0., 0.], -# [-360., -360., -360., ..., 0., 0., 0.]], \ No newline at end of file +Pr = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py index abff52e381..88cddb6573 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py @@ -1,3 +1,4 @@ +"""Fixes for rcm KNMI-RACMO22E driven by ICHEC-EC-EARTH.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py index 97abb80f26..9aeb0ebc02 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py @@ -1,11 +1,7 @@ - +"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by ICHEC-EC-EARTH.""" from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix -import numpy as np Tas = BaseFix -Pr = BaseFix - - - +Pr = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py index 860a814ba6..8a01e8e911 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py @@ -1,5 +1,7 @@ +"""Fixes for rcm SMHI-RCA4 driven by ICHEC-EC-EARTH.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix Pr = BaseFix + Tas = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py index a4a95eb8f2..ac06e8e4ab 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py @@ -1,15 +1,12 @@ - +"""Fixes for rcm DMI-HIRHAM driven by MOHC-HadGEM2.""" from esmvalcore.cmor.fix import Fix class Pr(Fix): - """Fixes for tas.""" + """Fixes for pr.""" def fix_metadata(self, cubes): """ - Add height (2m) coordinate. - - Fix also done for prw. - Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + Remove latitude and longitude attributes Parameters ---------- diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py index dfd992b2ab..1d1047e5e3 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py @@ -1,17 +1,7 @@ +"""Fixes for rcm GERICS-REMO2015 driven by MOHC-HadGEM2.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix Pr = BaseFix -Tas = BaseFix -# for historical + rcp45 in Pr Amon, longitudes are shifted - -# cubes[0].coord('longitude').points - cubes[1].coord('longitude').points -# masked_array( -# data=[[-360., -360., -360., ..., 0., 0., 0.], -# [-360., -360., -360., ..., 0., 0., 0.], -# [-360., -360., -360., ..., 0., 0., 0.], -# ..., -# [-360., -360., -360., ..., 0., 0., 0.], -# [-360., -360., -360., ..., 0., 0., 0.], -# [-360., -360., -360., ..., 0., 0., 0.]], \ No newline at end of file +Tas = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py index c100d08b15..e4df2b5fa0 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py @@ -1,17 +1,14 @@ - +"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by MOHC-HadGEM2.""" from esmvalcore.cmor.fix import Fix -import numpy as np class Tas(Fix): """Fixes for tas.""" def fix_metadata(self, cubes): """ - Add height (2m) coordinate. - - Fix also done for prw. - Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + Fix time long_name. + Fix latitude and longitude var_name. Parameters ---------- diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py index 860a814ba6..0babd14bca 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py @@ -1,5 +1,7 @@ +"""Fixes for rcm SMHI-RCA4 driven by MOHC-HadGEM2.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix Pr = BaseFix + Tas = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py index e68f39d014..c35330763d 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py @@ -1,20 +1,13 @@ -# height 2m a differents altures +"""Fixes for rcm GERICS-REMO2015 driven by NCC-NorESM1-M.""" from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.shared import add_scalar_height_coord -from cf_units import Unit -import iris -import numpy as np class Pr(Fix): - """Fixes for tas.""" + """Fixes for pr.""" def fix_metadata(self, cubes): """ - Add height (2m) coordinate. - - Fix also done for prw. - Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + Fix time long_name. Parameters ---------- diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py index e9b59eab12..22eb5145f4 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py @@ -1,3 +1,4 @@ +"""Fixes for rcm KNMI-RACMO22E driven by NCC-NorESM1-M.""" from .gerics_remo2015 import Pr as BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py index 97abb80f26..5d761759ed 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py @@ -1,11 +1,7 @@ - +"""Fixes for rcm HadREM3-GA7-05 driven by NCC-NorESM1-M.""" from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix -import numpy as np Tas = BaseFix -Pr = BaseFix - - - +Pr = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py index 4f11297652..4f5e520569 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py @@ -1,5 +1,7 @@ +"""Fixes for rcm SMHI-RCA4 driven by NCC-NorESM1-M.""" from .gerics_remo2015 import Pr as BaseFix Pr = BaseFix + Tas = BaseFix \ No newline at end of file From 3f2a08e8a66781698937846684f44dc7b2dba732 Mon Sep 17 00:00:00 2001 From: Pep Cos Date: Tue, 18 Oct 2022 16:59:49 +0200 Subject: [PATCH 15/75] delete unneeded fix --- .../cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py deleted file mode 100644 index 5905526fe0..0000000000 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py +++ /dev/null @@ -1,10 +0,0 @@ - -from esmvalcore.cmor.fix import Fix - -#rcp85 data does not have grid_lat and grid_lon defined -# Traceback (most recent call last): -# File "", line 1, in -# File "/home/b/b381943/ESMValCore/esmvalcore/preprocessor/_io.py", line 220, in _get_concatenation_error -# raise ValueError(f'Can not concatenate cubes: {msg}') -# ValueError: Can not concatenate cubes: failed to concatenate into a single cube. -# Dimension coordinates differ: grid_latitude, grid_longitude, time != time \ No newline at end of file From 4cc84e533fa4d1c455304fbf7b25f41b40c6b42d Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Oct 2022 07:50:53 +0200 Subject: [PATCH 16/75] Add more fixes --- .../cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py | 14 +--- .../mohc_hadrem3_ga7_05.py | 1 + .../ichec_ec_earth/clmcom_cclm4_8_17.py | 47 +---------- .../cordex/ichec_ec_earth/dmi_hirham5.py | 10 --- .../cordex/miroc_miroc5/clmcom_cclm4_8_17.py | 15 ++-- .../cordex/miroc_miroc5/gerics_remo2015.py | 2 +- .../cordex/miroc_miroc5/uhoh_wrf361h.py | 77 ------------------- .../cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py | 10 +-- .../cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py | 3 +- .../mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py | 3 +- .../cordex/mpi_m_mpi_esm_lr/uhoh_wrf361h.py | 2 - esmvalcore/cmor/check.py | 2 + 12 files changed, 23 insertions(+), 163 deletions(-) delete mode 100644 esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py delete mode 100644 esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py delete mode 100644 esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/uhoh_wrf361h.py diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py index e9018ca815..cef67f800e 100644 --- a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py @@ -1,4 +1,4 @@ -# height 2m a differents altures +"""Fixes for rcm CNRM-ALADIN63 driven by CNRM-CERFACS-CNRM-CM5.""" from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.shared import add_scalar_height_coord @@ -9,10 +9,7 @@ class Tas(Fix): def fix_metadata(self, cubes): """ - Add height (2m) coordinate. - - Fix also done for prw. - Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + Add height (2m) coordinate and correct long_name for time. Parameters ---------- @@ -33,14 +30,11 @@ def fix_metadata(self, cubes): return cubes class Pr(Fix): - """Fixes for tas.""" + """Fixes for pr.""" def fix_metadata(self, cubes): """ - Add height (2m) coordinate. - - Fix also done for prw. - Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + Correct long_name for time. Parameters ---------- diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py index b4e109c907..69cd0b7cd3 100644 --- a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py @@ -1,3 +1,4 @@ +"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by CNRM-CERFACS-CNRM-CM5.""" from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py index 92d1396298..99b237ee5d 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py @@ -1,45 +1,4 @@ -# height 2m a differents altures -from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.shared import add_scalar_height_coord +"""Fixes for rcm CLMcom-CCLM4-8-17 driven by ICHEC-EC-EARTH.""" +from ..miroc_miroc5.clmcom_cclm4_8_17 import AllVars as BaseFix -from cf_units import Unit -import iris -import numpy as np - -class AllVars(Fix): - """Fixes for tas.""" - - def fix_metadata(self, cubes): - """ - Add height (2m) coordinate. - - Fix also done for prw. - Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. - - Parameters - ---------- - cubes : iris.cube.CubeList - Input cubes. - - Returns - ------- - iris.cube.CubeList - - """ - for cube in cubes: - if cube.coord('time').units.calendar == 'gregorian': - cube.coord('time').units = Unit( - cube.coord('time').units.cftime_unit, - calendar='proleptic_gregorian' - ) - for coord in cube.coords(): - if coord.dtype in ['>f8', '>f4']: - coord.points = coord.core_points().astype( - np.float64, casting='same_kind') - if coord.bounds is not None: - coord.bounds = coord.core_bounds().astype( - np.float64, casting='same_kind') - # further issues with dtype, may be due because historical data lat/lon does not have bounds whereas scenario data has them - - - return cubes \ No newline at end of file +AllVars = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py deleted file mode 100644 index 5905526fe0..0000000000 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/dmi_hirham5.py +++ /dev/null @@ -1,10 +0,0 @@ - -from esmvalcore.cmor.fix import Fix - -#rcp85 data does not have grid_lat and grid_lon defined -# Traceback (most recent call last): -# File "", line 1, in -# File "/home/b/b381943/ESMValCore/esmvalcore/preprocessor/_io.py", line 220, in _get_concatenation_error -# raise ValueError(f'Can not concatenate cubes: {msg}') -# ValueError: Can not concatenate cubes: failed to concatenate into a single cube. -# Dimension coordinates differ: grid_latitude, grid_longitude, time != time \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py index 299ed8e561..8606b3f33e 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py @@ -1,9 +1,7 @@ -# height 2m a differents altures +"""Fixes for rcm CLMcom-CCLM4-8-17 driven by MIROC-MIROC5.""" from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.shared import add_scalar_height_coord from cf_units import Unit -import iris import numpy as np class AllVars(Fix): @@ -11,10 +9,11 @@ class AllVars(Fix): def fix_metadata(self, cubes): """ - Add height (2m) coordinate. + Set calendar to 'proleptic_gregorian' to avoid + concatenation issues between historical and + scenario runs. - Fix also done for prw. - Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. + Fix dtype value of coordinates and coordinate bounds. Parameters ---------- @@ -39,7 +38,9 @@ def fix_metadata(self, cubes): if coord.bounds is not None: coord.bounds = coord.core_bounds().astype( np.float64, casting='same_kind') - # further issues with dtype, may be due because historical data lat/lon does not have bounds whereas scenario data has them + + # Further issues appear, maybe because historical data lat/lon + # does not have bounds whereas scenario data has them. return cubes \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py index a29514edc8..1d3d99a6b7 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py @@ -1,4 +1,4 @@ -# height 2m a differents altures +"""Fixes for rcm GERICS-REMO2015 driven by MIROC-MIROC5.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix Pr = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py deleted file mode 100644 index 9836d2ea62..0000000000 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py +++ /dev/null @@ -1,77 +0,0 @@ -from esmvalcore.cmor.fix import Fix - -import iris - -class Tas(Fix): - """Fixes for tas.""" - - def fix_metadata(self, cubes): - """ - Add height (2m) coordinate. - - Fix also done for prw. - Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. - - Parameters - ---------- - cubes : iris.cube.CubeList - Input cubes. - - Returns - ------- - iris.cube.CubeList - - """ - squeezed = [] - for cube in cubes: - height = cube.coord('height') - if isinstance(height, iris.coords.DimCoord): - squeezed.append(iris.util.squeeze(cube)) - - if squeezed: - return squeezed - - return cubes - -# for tas / pr rcp85, lat and lon do not exist - - -# esmvalcore.cmor.check.CMORCheckError: There were errors in variable pr: -# longitude: does not exist -# latitude: does not exist -# pr: does not match coordinate rank -# in cube: -# precipitation_flux / (kg m-2 s-1) (time: 60; -- : 412; -- : 424) -# Dimension coordinates: -# time x - - -# Cell methods: -# mean time -# Attributes: -# CDO 'Climate Data Operators version 1.9.4 (http://mpimet.mpg.de/cdo)' -# CORDEX_domain 'EUR-11' -# Conventions 'CF-1.4' -# NCO '20180925' -# c3s_disclaimer 'This data has been produced in the context of PRINCIPLE/CORDEX4CDS project... -# contact 'viktoria.mohr@uni-hohenheim.de' -# driving_experiment 'MIROC-MIROC5, rcp85, r1i1p1' -# driving_experiment_name 'rcp85' -# driving_model_ensemble_member 'r1i1p1' -# driving_model_id 'MIROC-MIROC5' -# experiment 'rcp85' -# experiment_id 'rcp85' -# frequency 'mon' -# institute_id 'UHOH' -# institute_run_id 'hoh' -# institution 'Institute of Physics an Meteorology University of Hohenheim (UHOH), G... -# model_id 'UHOH-WRF361H' -# nco_openmp_thread_number 1 -# product 'output' -# project_id 'CORDEX' -# rcm_version_id 'v1' -# references 'https://www120.uni-hohenheim.de' -# source_file '/work/ik1017/C3SCORDEX/data/c3s-cordex/output/EUR-11/UHOH/MIROC-MIROC... -# loaded from file /work/ik1017/C3SCORDEX/data/c3s-cordex/output/EUR-11/UHOH/MIROC-MIROC5/rcp85/r1i1p1/UHOH-WRF361H/v1/mon/pr/v20180717/pr_EUR-11_MIROC-MIROC5_rcp85_r1i1p1_UHOH-WRF361H_v1_mon_200601-201012.nc -# 2022-10-11 10:04:10,988 UTC [3437717] INFO -# If you have a question or need help, please start a new discussion on https://github.com/ESMValGroup/ESMValTool/discussions -# If you suspect this is a bug, please open an issue on https://github.com/ESMValGroup/ESMValTool/issues -# To make it easier to find out what the problem is, please consider attaching the files run/recipe_*.yml and run/main_log_debug.txt from the output directory. diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py index 27825f8ba0..355f46bf91 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py @@ -1,13 +1,5 @@ -# 3hr data between historical and scenario cannot be concatenated because historical -# finishes at -# Cell(point=cftime.DatetimeGregorian(2005, 12, 31, 21, 0, 0, 0, has_year_zero=False), bound=None) -# and scenario finishes at -# Cell(point=cftime.DatetimeGregorian(2006, 1, 1, 3, 0, 0, 0, has_year_zero=False), bound=None) -# which is 6h apart instead of 3h - - +"""Fixes for rcm ICTP-RegCM4-6 driven by MPI-M-MPI-ESM-LR.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix - Pr = BaseFix Tas = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py index abff52e381..81550e4d2f 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py @@ -1,4 +1,5 @@ -from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix +"""Fixes for rcm KNMI-RACMO22E driven by MPI-M-MPI-ESM-LR.""" +from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix Pr = BaseFix \ No newline at end of file diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py index 97abb80f26..cdb0173147 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py @@ -1,8 +1,7 @@ +"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by MPI-M-MPI-ESM-LR.""" from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix -import numpy as np - Tas = BaseFix Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/uhoh_wrf361h.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/uhoh_wrf361h.py deleted file mode 100644 index d5800337a7..0000000000 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/uhoh_wrf361h.py +++ /dev/null @@ -1,2 +0,0 @@ -# Rounding errors between historical and scenario exp in grid_latitude and grid_longitude - diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 9691010988..b1ca33ebb1 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -1077,6 +1077,8 @@ def _get_cmor_checker(table, table, ', '.join(CMOR_TABLES))) cmor_table = CMOR_TABLES[table] + if table == 'CORDEX' and mip.endswith('hr'): + mip = mip.replace('hr', 'h') var_info = cmor_table.get_variable(mip, short_name) if var_info is None: var_info = CMOR_TABLES['custom'].get_variable(mip, short_name) From 301eed51e4ea0f3330ac5be5909bfb12de313e70 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Oct 2022 12:14:58 +0200 Subject: [PATCH 17/75] Remove unrelated files --- .../cmor/tables/custom/CMOR_siextent.dat | 23 ------- esmvalcore/preprocessor/_derive/siextent.py | 64 ------------------- 2 files changed, 87 deletions(-) delete mode 100644 esmvalcore/cmor/tables/custom/CMOR_siextent.dat delete mode 100644 esmvalcore/preprocessor/_derive/siextent.py diff --git a/esmvalcore/cmor/tables/custom/CMOR_siextent.dat b/esmvalcore/cmor/tables/custom/CMOR_siextent.dat deleted file mode 100644 index 736744595f..0000000000 --- a/esmvalcore/cmor/tables/custom/CMOR_siextent.dat +++ /dev/null @@ -1,23 +0,0 @@ -SOURCE: CMIP5 -!============ -variable_entry: siextent -!============ -modeling_realm: seaIce -!---------------------------------- -! Variable attributes: -!---------------------------------- -standard_name: -units: m2 -cell_methods: area: mean where sea time: mean -cell_measures: area: areacello -long_name: Sea Ice Extent -comment: -!---------------------------------- -! Additional variable information: -!---------------------------------- -dimensions: longitude latitude time -type: real -valid_min: -valid_max: -!---------------------------------- -! diff --git a/esmvalcore/preprocessor/_derive/siextent.py b/esmvalcore/preprocessor/_derive/siextent.py deleted file mode 100644 index d59335f8dd..0000000000 --- a/esmvalcore/preprocessor/_derive/siextent.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Derivation of variable `sithick`.""" -import logging - -import dask.array as da -import iris -from iris import Constraint - -from ._baseclass import DerivedVariableBase - -logger = logging.getLogger(__name__) - - -class DerivedVariable(DerivedVariableBase): - """Derivation of variable `siextent`.""" - - @staticmethod - def required(project): - """Declare the variables needed for derivation.""" - required = [ - { - 'short_name': 'sic', - 'optional': 'true' - }, - { - 'short_name': 'siconca', - 'optional': 'true' - }] - return required - - @staticmethod - def calculate(cubes): - """Compute sea ice extent. - - Returns an array of ones in every grid point where - the sea ice area fraction has values > 15 . - - Use in combination with the preprocessor - `area_statistics(operator='sum')` to weigh by the area and - compute global or regional sea ice extent values. - - Arguments - --------- - cubes: cubelist containing sea ice area fraction. - - Returns - ------- - Cube containing sea ice extent. - """ - try: - sic = cubes.extract_cube(Constraint(name='sic')) - except iris.exceptions.ConstraintMismatchError: - try: - sic = cubes.extract_cube(Constraint(name='siconca')) - except iris.exceptions.ConstraintMismatchError: - logger.error( - 'Derivation of siextent failed due to missing variables ' - 'sic and siconca.') - - ones = da.ones_like(sic) - siextent_data = da.ma.masked_where(sic.lazy_data() < 15., ones) - siextent = sic.copy(siextent_data) - siextent.units = 'm2' - - return siextent From 4d921ccff5762b98ef256a2bcdcb504833b86e99 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Oct 2022 12:21:55 +0200 Subject: [PATCH 18/75] Delete unrelated test --- .../preprocessor/_derive/test_siextent.py | 108 ------------------ 1 file changed, 108 deletions(-) delete mode 100644 tests/unit/preprocessor/_derive/test_siextent.py diff --git a/tests/unit/preprocessor/_derive/test_siextent.py b/tests/unit/preprocessor/_derive/test_siextent.py deleted file mode 100644 index 0b21c4b216..0000000000 --- a/tests/unit/preprocessor/_derive/test_siextent.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Test derivation of `ohc`.""" -import cf_units -import iris -import numpy as np -import pytest - -import esmvalcore.preprocessor._derive.siextent as siextent - - -@pytest.fixture -def cubes_sic(): - sic_name = 'sea_ice_area_fraction' - time_coord = iris.coords.DimCoord([0., 1., 2.], - standard_name='time') - sic_cube = iris.cube.Cube([[[20, 10], [10, 10]], - [[10, 10], [10, 10]], - [[10, 10], [10, 10]]], - units='%', - standard_name=sic_name, - var_name='sic', - dim_coords_and_dims=[(time_coord, 0)]) - return iris.cube.CubeList([sic_cube]) - - -@pytest.fixture -def cubes_siconca(): - sic_name = 'sea_ice_area_fraction' - time_coord = iris.coords.DimCoord([0., 1., 2.], - standard_name='time') - sic_cube = iris.cube.Cube([[[20, 10], [10, 10]], - [[10, 10], [10, 10]], - [[10, 10], [10, 10]]], - units='%', - standard_name=sic_name, - var_name='siconca', - dim_coords_and_dims=[(time_coord, 0)]) - return iris.cube.CubeList([sic_cube]) - - -@pytest.fixture -def cubes(): - sic_name = 'sea_ice_area_fraction' - time_coord = iris.coords.DimCoord([0., 1., 2.], - standard_name='time') - sic_cube = iris.cube.Cube([[[20, 10], [10, 10]], - [[10, 10], [10, 10]], - [[10, 10], [10, 10]]], - units='%', - standard_name=sic_name, - var_name='sic', - dim_coords_and_dims=[(time_coord, 0)]) - siconca_cube = iris.cube.Cube([[[20, 10], [10, 10]], - [[10, 10], [10, 10]], - [[10, 10], [10, 10]]], - units='%', - standard_name=sic_name, - var_name='siconca', - dim_coords_and_dims=[(time_coord, 0)]) - return iris.cube.CubeList([sic_cube, siconca_cube]) - - -def test_siextent_calculation_sic(cubes_sic): - """Test function ``calculate`` when sic is available.""" - derived_var = siextent.DerivedVariable() - out_cube = derived_var.calculate(cubes_sic) - assert out_cube.units == cf_units.Unit('m2') - out_data = out_cube.data - expected = np.ma.ones_like(cubes_sic[0].data) - expected.mask = True - expected[0][0][0] = 1. - np.testing.assert_array_equal(out_data.mask, expected.mask) - np.testing.assert_array_equal(out_data[0][0][0], expected[0][0][0]) - - -def test_siextent_calculation_siconca(cubes_siconca): - """Test function ``calculate`` when siconca is available.""" - derived_var = siextent.DerivedVariable() - out_cube = derived_var.calculate(cubes_siconca) - assert out_cube.units == cf_units.Unit('m2') - out_data = out_cube.data - expected = np.ma.ones_like(cubes_siconca[0].data) - expected.mask = True - expected[0][0][0] = 1. - np.testing.assert_array_equal(out_data.mask, expected.mask) - np.testing.assert_array_equal(out_data[0][0][0], expected[0][0][0]) - - -def test_siextent_calculation(cubes): - """Test function ``calculate`` when sic and siconca are available.""" - derived_var = siextent.DerivedVariable() - out_cube = derived_var.calculate(cubes) - assert out_cube.units == cf_units.Unit('m2') - out_data = out_cube.data - expected = np.ma.ones_like(cubes[0].data) - expected.mask = True - expected[0][0][0] = 1. - np.testing.assert_array_equal(out_data.mask, expected.mask) - np.testing.assert_array_equal(out_data[0][0][0], expected[0][0][0]) - - -def test_siextent_required(): - """Test function ``required``.""" - derived_var = siextent.DerivedVariable() - output = derived_var.required(None) - assert output == [ - {'short_name': 'sic', 'optional': 'true'}, - {'short_name': 'siconca', 'optional': 'true'} - ] From c3b1bc113a98abbbf0d5ee2aee5e82acadbbb4cd Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Oct 2022 14:45:27 +0200 Subject: [PATCH 19/75] Remove dataset that may not need fix --- .../cmor/_fixes/cordex/mohc_hadgem2_es/ictp_regcm4_6.py | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/ictp_regcm4_6.py diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/ictp_regcm4_6.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/ictp_regcm4_6.py deleted file mode 100644 index 3a732e783c..0000000000 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/ictp_regcm4_6.py +++ /dev/null @@ -1,6 +0,0 @@ -# 3hr data between historical and scenario cannot be concatenated because historical -# finishes at -# Cell(point=cftime.DatetimeGregorian(2005, 12, 31, 21, 0, 0, 0, has_year_zero=False), bound=None) -# and scenario finishes at -# Cell(point=cftime.DatetimeGregorian(2006, 1, 1, 3, 0, 0, 0, has_year_zero=False), bound=None) -# which is 6h apart instead of 3h From 43e3139bf8b7d1f13d5c2edf033a5a87a8ba4492 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Oct 2022 14:55:02 +0200 Subject: [PATCH 20/75] Fix style issues --- .../_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py | 4 +++- .../cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py | 2 +- .../_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py | 2 +- .../cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py | 3 +-- .../cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py | 3 +-- .../_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py | 3 +-- .../cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py | 3 +-- .../cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py | 3 ++- .../cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py | 2 +- .../cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py | 7 ++++--- .../_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py | 3 +-- .../_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py | 9 +++++---- .../cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py | 3 +-- .../cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py | 3 ++- .../cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py | 3 +-- .../cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py | 4 ---- .../cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py | 2 +- .../cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py | 3 +-- .../_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py | 3 +-- esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py | 3 +-- 20 files changed, 30 insertions(+), 38 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py index cef67f800e..dfdc7153e3 100644 --- a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py @@ -4,6 +4,7 @@ import numpy as np + class Tas(Fix): """Fixes for tas.""" @@ -29,6 +30,7 @@ def fix_metadata(self, cubes): return cubes + class Pr(Fix): """Fixes for pr.""" @@ -49,4 +51,4 @@ def fix_metadata(self, cubes): for cube in cubes: cube.coord('time').long_name = 'time' - return cubes \ No newline at end of file + return cubes diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py index 69cd0b7cd3..12b5a65bda 100644 --- a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py @@ -3,4 +3,4 @@ Tas = BaseFix -Pr = BaseFix \ No newline at end of file +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py index 99b237ee5d..378fa37793 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py @@ -1,4 +1,4 @@ """Fixes for rcm CLMcom-CCLM4-8-17 driven by ICHEC-EC-EARTH.""" from ..miroc_miroc5.clmcom_cclm4_8_17 import AllVars as BaseFix -AllVars = BaseFix \ No newline at end of file +AllVars = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py index 58bc498941..f0e1b470b6 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py @@ -1,5 +1,4 @@ """Fixes for rcm GERICS-REMO2015 driven by ICHEC-EC-EARTH.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix - -Pr = BaseFix \ No newline at end of file +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py index 88cddb6573..ac0643eb1b 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py @@ -1,5 +1,4 @@ """Fixes for rcm KNMI-RACMO22E driven by ICHEC-EC-EARTH.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix - -Pr = BaseFix \ No newline at end of file +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py index 9aeb0ebc02..ca0a22a1ef 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py @@ -1,7 +1,6 @@ """Fixes for rcm MOHC-HadREM3-GA7-05 driven by ICHEC-EC-EARTH.""" from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix - Tas = BaseFix -Pr = BaseFix \ No newline at end of file +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py index 8a01e8e911..fcb05ba0e7 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py @@ -1,7 +1,6 @@ """Fixes for rcm SMHI-RCA4 driven by ICHEC-EC-EARTH.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix - Pr = BaseFix -Tas = BaseFix \ No newline at end of file +Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py index 8606b3f33e..4e547067bf 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py @@ -4,6 +4,7 @@ from cf_units import Unit import numpy as np + class AllVars(Fix): """Fixes for tas.""" @@ -43,4 +44,4 @@ def fix_metadata(self, cubes): # does not have bounds whereas scenario data has them. - return cubes \ No newline at end of file + return cubes diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py index 1d3d99a6b7..ecfd2cdb9b 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py @@ -1,4 +1,4 @@ """Fixes for rcm GERICS-REMO2015 driven by MIROC-MIROC5.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix -Pr = BaseFix \ No newline at end of file +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py index ac06e8e4ab..6c1804b9d8 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py @@ -1,6 +1,7 @@ """Fixes for rcm DMI-HIRHAM driven by MOHC-HadGEM2.""" from esmvalcore.cmor.fix import Fix + class Pr(Fix): """Fixes for pr.""" @@ -19,7 +20,7 @@ def fix_metadata(self, cubes): """ for cube in cubes: - cube.coord('latitude').attributes = {} - cube.coord('longitude').attributes = {} + cube.coord('latitude').attributes = {} + cube.coord('longitude').attributes = {} - return cubes \ No newline at end of file + return cubes diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py index 1d1047e5e3..95169eb18c 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py @@ -1,7 +1,6 @@ """Fixes for rcm GERICS-REMO2015 driven by MOHC-HadGEM2.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix - Pr = BaseFix -Tas = BaseFix \ No newline at end of file +Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py index e4df2b5fa0..7c03d33f7d 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py @@ -21,10 +21,11 @@ def fix_metadata(self, cubes): """ for cube in cubes: - cube.coord('latitude').var_name = 'lat' - cube.coord('longitude').var_name = 'lon' - cube.coord('time').long_name = 'time' + cube.coord('latitude').var_name = 'lat' + cube.coord('longitude').var_name = 'lon' + cube.coord('time').long_name = 'time' return cubes -Pr = Tas \ No newline at end of file + +Pr = Tas diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py index 0babd14bca..8ed38fc869 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py @@ -1,7 +1,6 @@ """Fixes for rcm SMHI-RCA4 driven by MOHC-HadGEM2.""" from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix - Pr = BaseFix -Tas = BaseFix \ No newline at end of file +Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py index 355f46bf91..1745034916 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py @@ -2,4 +2,5 @@ from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix Pr = BaseFix -Tas = BaseFix \ No newline at end of file + +Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py index 81550e4d2f..20e2bd8602 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py @@ -1,5 +1,4 @@ """Fixes for rcm KNMI-RACMO22E driven by MPI-M-MPI-ESM-LR.""" - from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix -Pr = BaseFix \ No newline at end of file +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py index cdb0173147..49775a50a2 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py @@ -1,10 +1,6 @@ """Fixes for rcm MOHC-HadREM3-GA7-05 driven by MPI-M-MPI-ESM-LR.""" - from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix Tas = BaseFix Pr = BaseFix - - - diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py index c35330763d..db84fa9bc1 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py @@ -22,4 +22,4 @@ def fix_metadata(self, cubes): for cube in cubes: cube.coord('time').long_name = 'time' - return cubes \ No newline at end of file + return cubes diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py index 22eb5145f4..02d2abaf54 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py @@ -1,5 +1,4 @@ """Fixes for rcm KNMI-RACMO22E driven by NCC-NorESM1-M.""" from .gerics_remo2015 import Pr as BaseFix - -Pr = BaseFix \ No newline at end of file +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py index 5d761759ed..1ec5a6cbb0 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py @@ -1,7 +1,6 @@ """Fixes for rcm HadREM3-GA7-05 driven by NCC-NorESM1-M.""" from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix - Tas = BaseFix -Pr = BaseFix \ No newline at end of file +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py index 4f5e520569..ed0be71080 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py @@ -1,7 +1,6 @@ """Fixes for rcm SMHI-RCA4 driven by NCC-NorESM1-M.""" from .gerics_remo2015 import Pr as BaseFix - Pr = BaseFix -Tas = BaseFix \ No newline at end of file +Tas = BaseFix From 7325d625eb270b1a5e1bf6b6c06cf28fa6f91ad6 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Oct 2022 16:59:35 +0200 Subject: [PATCH 21/75] Fix style issues --- .../cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py | 5 +---- .../cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py index 4e547067bf..1e48c1bf24 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py @@ -39,9 +39,6 @@ def fix_metadata(self, cubes): if coord.bounds is not None: coord.bounds = coord.core_bounds().astype( np.float64, casting='same_kind') - - # Further issues appear, maybe because historical data lat/lon + # Further issues appear, maybe because historical data lat/lon # does not have bounds whereas scenario data has them. - - return cubes diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py index db84fa9bc1..db21da39cb 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py @@ -20,6 +20,6 @@ def fix_metadata(self, cubes): """ for cube in cubes: - cube.coord('time').long_name = 'time' + cube.coord('time').long_name = 'time' return cubes From bce8fc60e99346a220caf84d74ca750bb80765f2 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Oct 2022 17:30:20 +0200 Subject: [PATCH 22/75] Reorder HadREM fixes --- .../mohc_hadrem3_ga7_05.py | 2 +- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 28 +++++++++++++++++ .../ichec_ec_earth/mohc_hadrem3_ga7_05.py | 2 +- .../mohc_hadgem2_es/mohc_hadrem3_ga7_05.py | 31 ++----------------- .../mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py | 2 +- .../ncc_noresm1_m/mohc_hadrem3_ga7_05.py | 2 +- 6 files changed, 35 insertions(+), 32 deletions(-) create mode 100644 esmvalcore/cmor/_fixes/cordex/cordex_fixes.py diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py index 12b5a65bda..e1c146eb27 100644 --- a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py @@ -1,5 +1,5 @@ """Fixes for rcm MOHC-HadREM3-GA7-05 driven by CNRM-CERFACS-CNRM-CM5.""" -from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix +from ..cordex_fixes import MOHCHadREM3GA705 as BaseFix Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py new file mode 100644 index 0000000000..a9e796ddd8 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -0,0 +1,28 @@ +"""Fixes that are shared between datasets and drivers.""" +from ..fix import Fix + + +class MOHCHadREM3GA705(Fix): + """General fix for MOHC-HadREM3-GA7-05.""" + + def fix_metadata(self, cubes): + """ + Fix time long_name. + Fix latitude and longitude var_name. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + cube.coord('latitude').var_name = 'lat' + cube.coord('longitude').var_name = 'lon' + cube.coord('time').long_name = 'time' + + return cubes diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py index ca0a22a1ef..cbf2e28207 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py @@ -1,5 +1,5 @@ """Fixes for rcm MOHC-HadREM3-GA7-05 driven by ICHEC-EC-EARTH.""" -from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix +from ..cordex_fixes import MOHCHadREM3GA705 as BaseFix Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py index 7c03d33f7d..f32b791261 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py @@ -1,31 +1,6 @@ """Fixes for rcm MOHC-HadREM3-GA7-05 driven by MOHC-HadGEM2.""" -from esmvalcore.cmor.fix import Fix +from ..cordex_fixes import MOHCHadREM3GA705 as BaseFix +Tas = BaseFix -class Tas(Fix): - """Fixes for tas.""" - - def fix_metadata(self, cubes): - """ - Fix time long_name. - Fix latitude and longitude var_name. - - Parameters - ---------- - cubes : iris.cube.CubeList - Input cubes. - - Returns - ------- - iris.cube.CubeList - - """ - for cube in cubes: - cube.coord('latitude').var_name = 'lat' - cube.coord('longitude').var_name = 'lon' - cube.coord('time').long_name = 'time' - - return cubes - - -Pr = Tas +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py index 49775a50a2..8f7ccb4fe8 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py @@ -1,5 +1,5 @@ """Fixes for rcm MOHC-HadREM3-GA7-05 driven by MPI-M-MPI-ESM-LR.""" -from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix +from ..cordex_fixes import MOHCHadREM3GA705 as BaseFix Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py index 1ec5a6cbb0..1132587a42 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py @@ -1,5 +1,5 @@ """Fixes for rcm HadREM3-GA7-05 driven by NCC-NorESM1-M.""" -from ..mohc_hadgem2_es.mohc_hadrem3_ga7_05 import Tas as BaseFix +from ..cordex_fixes import MOHCHadREM3GA705 as BaseFix Tas = BaseFix From 3df519b210d95a7b161db7fb526d6bd7c19468ba Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Oct 2022 17:36:38 +0200 Subject: [PATCH 23/75] Generalise time coord fix --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 23 +++++++++++++++++ .../cordex/ichec_ec_earth/gerics_remo2015.py | 2 +- .../cordex/ichec_ec_earth/knmi_racmo22e.py | 2 +- .../_fixes/cordex/ichec_ec_earth/smhi_rca4.py | 2 +- .../cordex/miroc_miroc5/gerics_remo2015.py | 2 +- .../cordex/mohc_hadgem2_es/gerics_remo2015.py | 2 +- .../cordex/mohc_hadgem2_es/smhi_rca4.py | 2 +- .../cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py | 2 +- .../cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py | 2 +- .../cordex/ncc_noresm1_m/gerics_remo2015.py | 25 ++----------------- .../cordex/ncc_noresm1_m/knmi_racmo22e.py | 2 +- .../_fixes/cordex/ncc_noresm1_m/smhi_rca4.py | 2 +- 12 files changed, 35 insertions(+), 33 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index a9e796ddd8..6430250be3 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -26,3 +26,26 @@ def fix_metadata(self, cubes): cube.coord('time').long_name = 'time' return cubes + + +class TimeLongName(Fix): + """Fixes for time coordinate.""" + + def fix_metadata(self, cubes): + """ + Fix time long_name. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + cube.coord('time').long_name = 'time' + + return cubes diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py index f0e1b470b6..edbc705867 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py @@ -1,4 +1,4 @@ """Fixes for rcm GERICS-REMO2015 driven by ICHEC-EC-EARTH.""" -from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix +from ..cordex_fixes import TimeLongName as BaseFix Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py index ac0643eb1b..340f952b96 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py @@ -1,4 +1,4 @@ """Fixes for rcm KNMI-RACMO22E driven by ICHEC-EC-EARTH.""" -from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix +from ..cordex_fixes import TimeLongName as BaseFix Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py index fcb05ba0e7..3ae5023e12 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py @@ -1,5 +1,5 @@ """Fixes for rcm SMHI-RCA4 driven by ICHEC-EC-EARTH.""" -from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix +from ..cordex_fixes import TimeLongName as BaseFix Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py index ecfd2cdb9b..262e35bf03 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py @@ -1,4 +1,4 @@ """Fixes for rcm GERICS-REMO2015 driven by MIROC-MIROC5.""" -from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix +from ..cordex_fixes import TimeLongName as BaseFix Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py index 95169eb18c..4b4fae813c 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py @@ -1,5 +1,5 @@ """Fixes for rcm GERICS-REMO2015 driven by MOHC-HadGEM2.""" -from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix +from ..cordex_fixes import TimeLongName as BaseFix Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py index 8ed38fc869..2419a6b820 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py @@ -1,5 +1,5 @@ """Fixes for rcm SMHI-RCA4 driven by MOHC-HadGEM2.""" -from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix +from ..cordex_fixes import TimeLongName as BaseFix Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py index 1745034916..47fcbe648b 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py @@ -1,5 +1,5 @@ """Fixes for rcm ICTP-RegCM4-6 driven by MPI-M-MPI-ESM-LR.""" -from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix +from ..cordex_fixes import TimeLongName as BaseFix Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py index 20e2bd8602..5413e086ea 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py @@ -1,4 +1,4 @@ """Fixes for rcm KNMI-RACMO22E driven by MPI-M-MPI-ESM-LR.""" -from ..ncc_noresm1_m.gerics_remo2015 import Pr as BaseFix +from ..cordex_fixes import TimeLongName as BaseFix Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py index db21da39cb..f880b06e1c 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py @@ -1,25 +1,4 @@ """Fixes for rcm GERICS-REMO2015 driven by NCC-NorESM1-M.""" -from esmvalcore.cmor.fix import Fix +from ..cordex_fixes import TimeLongName as BaseFix - -class Pr(Fix): - """Fixes for pr.""" - - def fix_metadata(self, cubes): - """ - Fix time long_name. - - Parameters - ---------- - cubes : iris.cube.CubeList - Input cubes. - - Returns - ------- - iris.cube.CubeList - - """ - for cube in cubes: - cube.coord('time').long_name = 'time' - - return cubes +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py index 02d2abaf54..9c2baa581b 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py @@ -1,4 +1,4 @@ """Fixes for rcm KNMI-RACMO22E driven by NCC-NorESM1-M.""" -from .gerics_remo2015 import Pr as BaseFix +from ..cordex_fixes import TimeLongName as BaseFix Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py index ed0be71080..cd95a32f11 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py @@ -1,5 +1,5 @@ """Fixes for rcm SMHI-RCA4 driven by NCC-NorESM1-M.""" -from .gerics_remo2015 import Pr as BaseFix +from ..cordex_fixes import TimeLongName as BaseFix Pr = BaseFix From 2417b9da8c31602c8a7da38d45be039ab4b3d5df Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Oct 2022 18:05:11 +0200 Subject: [PATCH 24/75] Generalise CLM com fix --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 44 ++++++++++++++++++- .../ichec_ec_earth/clmcom_cclm4_8_17.py | 2 +- .../_fixes/cordex/miroc_miroc5/__init__.py | 1 + .../cordex/miroc_miroc5/clmcom_cclm4_8_17.py | 38 +--------------- 4 files changed, 47 insertions(+), 38 deletions(-) create mode 100644 esmvalcore/cmor/_fixes/cordex/miroc_miroc5/__init__.py diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 6430250be3..75f8483d6e 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -1,5 +1,8 @@ """Fixes that are shared between datasets and drivers.""" -from ..fix import Fix +from cf_units import Unit +import numpy as np + +from esmvalcore.cmor.fix import Fix class MOHCHadREM3GA705(Fix): @@ -49,3 +52,42 @@ def fix_metadata(self, cubes): cube.coord('time').long_name = 'time' return cubes + + +class CLMcomCCLM4817(Fix): + """Fixes for CLMcom-CCLM4-8-17.""" + + def fix_metadata(self, cubes): + """ + Set calendar to 'proleptic_gregorian' to avoid + concatenation issues between historical and + scenario runs. + + Fix dtype value of coordinates and coordinate bounds. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coord('time').units.calendar == 'gregorian': + cube.coord('time').units = Unit( + 'days since 1850-1-1 00:00:00', + calendar='proleptic_gregorian' + ) + for coord in cube.coords(): + if coord.dtype in ['>f8', '>f4']: + coord.points = coord.core_points().astype( + np.float64, casting='same_kind') + if coord.bounds is not None: + coord.bounds = coord.core_bounds().astype( + np.float64, casting='same_kind') + # Further issues appear, maybe because historical data lat/lon + # does not have bounds whereas scenario data has them. + return cubes diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py index 378fa37793..34b710e5dd 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py @@ -1,4 +1,4 @@ """Fixes for rcm CLMcom-CCLM4-8-17 driven by ICHEC-EC-EARTH.""" -from ..miroc_miroc5.clmcom_cclm4_8_17 import AllVars as BaseFix +from ..cordex_fixes import CLMcomCCLM4817 as BaseFix AllVars = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/__init__.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/__init__.py new file mode 100644 index 0000000000..093969370f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/__init__.py @@ -0,0 +1 @@ +"""Fixes for CORDEX data.""" diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py index 1e48c1bf24..71945f9ce2 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py @@ -5,40 +5,6 @@ import numpy as np -class AllVars(Fix): - """Fixes for tas.""" +from ..cordex_fixes import CLMcomCCLM4817 as BaseFix - def fix_metadata(self, cubes): - """ - Set calendar to 'proleptic_gregorian' to avoid - concatenation issues between historical and - scenario runs. - - Fix dtype value of coordinates and coordinate bounds. - - Parameters - ---------- - cubes : iris.cube.CubeList - Input cubes. - - Returns - ------- - iris.cube.CubeList - - """ - for cube in cubes: - if cube.coord('time').units.calendar == 'gregorian': - cube.coord('time').units = Unit( - 'days since 1850-1-1 00:00:00', - calendar='proleptic_gregorian' - ) - for coord in cube.coords(): - if coord.dtype in ['>f8', '>f4']: - coord.points = coord.core_points().astype( - np.float64, casting='same_kind') - if coord.bounds is not None: - coord.bounds = coord.core_bounds().astype( - np.float64, casting='same_kind') - # Further issues appear, maybe because historical data lat/lon - # does not have bounds whereas scenario data has them. - return cubes +AllVars = BaseFix From 02e795abf6f94a95eb635a98fbda19f7abe04c8e Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 19 Oct 2022 18:07:01 +0200 Subject: [PATCH 25/75] Fix imports --- .../cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py index 71945f9ce2..1a2fcf8d47 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py @@ -1,10 +1,4 @@ """Fixes for rcm CLMcom-CCLM4-8-17 driven by MIROC-MIROC5.""" -from esmvalcore.cmor.fix import Fix - -from cf_units import Unit -import numpy as np - - from ..cordex_fixes import CLMcomCCLM4817 as BaseFix AllVars = BaseFix From ca921ace69830dc70fd8bb58f06c350f0fc27816 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Oct 2022 11:52:49 +0200 Subject: [PATCH 26/75] Change imports --- .../cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py | 3 ++- esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py | 3 ++- esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py | 1 - .../cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py | 3 ++- .../cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py | 3 ++- esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py | 3 ++- .../cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py | 3 ++- .../cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py | 3 ++- .../cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py | 3 ++- esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py | 3 ++- esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py | 3 ++- .../cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py | 3 ++- esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py | 3 ++- 13 files changed, 24 insertions(+), 13 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py index 1a2fcf8d47..51d645f591 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py @@ -1,4 +1,5 @@ """Fixes for rcm CLMcom-CCLM4-8-17 driven by MIROC-MIROC5.""" -from ..cordex_fixes import CLMcomCCLM4817 as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + CLMcomCCLM4817 as BaseFix) AllVars = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py index 262e35bf03..9f75b595f9 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py @@ -1,4 +1,5 @@ """Fixes for rcm GERICS-REMO2015 driven by MIROC-MIROC5.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py index 6c1804b9d8..1a13071213 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py @@ -1,7 +1,6 @@ """Fixes for rcm DMI-HIRHAM driven by MOHC-HadGEM2.""" from esmvalcore.cmor.fix import Fix - class Pr(Fix): """Fixes for pr.""" diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py index 4b4fae813c..85bb2e352c 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py @@ -1,5 +1,6 @@ """Fixes for rcm GERICS-REMO2015 driven by MOHC-HadGEM2.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py index f32b791261..a2b791ceaa 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py @@ -1,5 +1,6 @@ """Fixes for rcm MOHC-HadREM3-GA7-05 driven by MOHC-HadGEM2.""" -from ..cordex_fixes import MOHCHadREM3GA705 as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + MOHCHadREM3GA705 as BaseFix) Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py index 2419a6b820..8108e238b8 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py @@ -1,5 +1,6 @@ """Fixes for rcm SMHI-RCA4 driven by MOHC-HadGEM2.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py index 47fcbe648b..9a92874b03 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py @@ -1,5 +1,6 @@ """Fixes for rcm ICTP-RegCM4-6 driven by MPI-M-MPI-ESM-LR.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py index 5413e086ea..ab944112b3 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py @@ -1,4 +1,5 @@ """Fixes for rcm KNMI-RACMO22E driven by MPI-M-MPI-ESM-LR.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py index 8f7ccb4fe8..0abfc3dff1 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py @@ -1,5 +1,6 @@ """Fixes for rcm MOHC-HadREM3-GA7-05 driven by MPI-M-MPI-ESM-LR.""" -from ..cordex_fixes import MOHCHadREM3GA705 as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + MOHCHadREM3GA705 as BaseFix) Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py index f880b06e1c..b1d118cd38 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py @@ -1,4 +1,5 @@ """Fixes for rcm GERICS-REMO2015 driven by NCC-NorESM1-M.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py index 9c2baa581b..cc2440a2b8 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py @@ -1,4 +1,5 @@ """Fixes for rcm KNMI-RACMO22E driven by NCC-NorESM1-M.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py index 1132587a42..1aa2d11b1b 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py @@ -1,5 +1,6 @@ """Fixes for rcm HadREM3-GA7-05 driven by NCC-NorESM1-M.""" -from ..cordex_fixes import MOHCHadREM3GA705 as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + MOHCHadREM3GA705 as BaseFix) Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py index cd95a32f11..5c9059f74a 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py @@ -1,5 +1,6 @@ """Fixes for rcm SMHI-RCA4 driven by NCC-NorESM1-M.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix From d804e67b60fd854c9024218d9ce602a29bc1925f Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Oct 2022 11:55:34 +0200 Subject: [PATCH 27/75] Fix style issues --- esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py index 1a13071213..6c1804b9d8 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py @@ -1,6 +1,7 @@ """Fixes for rcm DMI-HIRHAM driven by MOHC-HadGEM2.""" from esmvalcore.cmor.fix import Fix + class Pr(Fix): """Fixes for pr.""" From 61e15bb121c08120c3332e5d773b5588d35cc70a Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Oct 2022 14:25:08 +0200 Subject: [PATCH 28/75] Change imports in more files --- .../cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py | 3 ++- .../cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py | 3 ++- esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py | 3 ++- .../cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py | 3 ++- esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py index 34b710e5dd..883e38cecd 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py @@ -1,4 +1,5 @@ """Fixes for rcm CLMcom-CCLM4-8-17 driven by ICHEC-EC-EARTH.""" -from ..cordex_fixes import CLMcomCCLM4817 as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + CLMcomCCLM4817 as BaseFix) AllVars = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py index edbc705867..2f6b9a4d62 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py @@ -1,4 +1,5 @@ """Fixes for rcm GERICS-REMO2015 driven by ICHEC-EC-EARTH.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py index 340f952b96..96fd620afe 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py @@ -1,4 +1,5 @@ """Fixes for rcm KNMI-RACMO22E driven by ICHEC-EC-EARTH.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py index cbf2e28207..15b643aa36 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py @@ -1,5 +1,6 @@ """Fixes for rcm MOHC-HadREM3-GA7-05 driven by ICHEC-EC-EARTH.""" -from ..cordex_fixes import MOHCHadREM3GA705 as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + MOHCHadREM3GA705 as BaseFix) Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py index 3ae5023e12..de1d8d6536 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py @@ -1,5 +1,6 @@ """Fixes for rcm SMHI-RCA4 driven by ICHEC-EC-EARTH.""" -from ..cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) Pr = BaseFix From 05d9b2833b698aead927e84cdf2fdd08a7bb84f0 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Oct 2022 14:35:15 +0200 Subject: [PATCH 29/75] Change more imports --- .../_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py index e1c146eb27..fd31e9b8a6 100644 --- a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py @@ -1,5 +1,6 @@ """Fixes for rcm MOHC-HadREM3-GA7-05 driven by CNRM-CERFACS-CNRM-CM5.""" -from ..cordex_fixes import MOHCHadREM3GA705 as BaseFix +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + MOHCHadREM3GA705 as BaseFix) Tas = BaseFix From 870aa8644043f9001aa0e91d6e88b2b04b30a1f1 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Oct 2022 16:58:33 +0200 Subject: [PATCH 30/75] Improve test coverage of check.py --- tests/unit/cmor/test_cmor_check.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index fe965b5981..ebedb1d597 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -11,7 +11,9 @@ import numpy as np from cf_units import Unit -from esmvalcore.cmor.check import CheckLevels, CMORCheck, CMORCheckError +from esmvalcore.cmor.check import ( + CheckLevels, CMORCheck, + CMORCheckError, _get_cmor_checker) class VariableInfoMock: @@ -1075,6 +1077,12 @@ def test_no_time_bounds(self): self._check_cube(automatic_fixes=True) guessed_bounds = self.cube.coord('time').bounds assert guessed_bounds is None + + def test_hr_mip_cordex(self): + """Test hourly CORDEX tables are found.""" + checker = _get_cmor_checker('CORDEX', '3h', 'tas', '3hr') + assert checker(self.cube)._cmor_var.short_name == 'tas' + assert checker(self.cube)._cmor_var.frequency == '3hr' def _check_fails_on_data(self): checker = CMORCheck(self.cube, self.var_info) From 0f6108e8d8adc4367b97ab8e3a08cf5124f1def2 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Oct 2022 17:01:47 +0200 Subject: [PATCH 31/75] Remove whitespace --- tests/unit/cmor/test_cmor_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index ebedb1d597..69395298a7 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -12,7 +12,7 @@ from cf_units import Unit from esmvalcore.cmor.check import ( - CheckLevels, CMORCheck, + CheckLevels, CMORCheck, CMORCheckError, _get_cmor_checker) @@ -1077,7 +1077,7 @@ def test_no_time_bounds(self): self._check_cube(automatic_fixes=True) guessed_bounds = self.cube.coord('time').bounds assert guessed_bounds is None - + def test_hr_mip_cordex(self): """Test hourly CORDEX tables are found.""" checker = _get_cmor_checker('CORDEX', '3h', 'tas', '3hr') From 5e89d0fbb2f3e1b0edab26d0408086f6f5c9d9ef Mon Sep 17 00:00:00 2001 From: sloosvel Date: Thu, 20 Oct 2022 17:10:52 +0200 Subject: [PATCH 32/75] Fix test --- tests/unit/cmor/test_cmor_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index 69395298a7..4e17f9ca99 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -1080,7 +1080,7 @@ def test_no_time_bounds(self): def test_hr_mip_cordex(self): """Test hourly CORDEX tables are found.""" - checker = _get_cmor_checker('CORDEX', '3h', 'tas', '3hr') + checker = _get_cmor_checker('CORDEX', '3hr', 'tas', '3hr') assert checker(self.cube)._cmor_var.short_name == 'tas' assert checker(self.cube)._cmor_var.frequency == '3hr' From 729151d7b6576c4d76b67fa4080de39c5065f5f5 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 21 Oct 2022 16:56:46 +0200 Subject: [PATCH 33/75] Improve test coverage in fix.py --- tests/integration/cmor/_fixes/test_fix.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/cmor/_fixes/test_fix.py b/tests/integration/cmor/_fixes/test_fix.py index 5ffc4ca868..69cb446c09 100644 --- a/tests/integration/cmor/_fixes/test_fix.py +++ b/tests/integration/cmor/_fixes/test_fix.py @@ -12,6 +12,8 @@ from esmvalcore.cmor._fixes.cmip5.canesm2 import FgCo2 from esmvalcore.cmor._fixes.cmip5.cesm1_bgc import Gpp from esmvalcore.cmor._fixes.cmip6.cesm2 import Omon, Tos +from esmvalcore.cmor._fixes.cordex.cnrm_cerfacs_cnrm_cm5.cnrm_aladin63 import ( + Tas) from esmvalcore.cmor.fix import Fix @@ -32,6 +34,16 @@ def test_get_fix_case_insensitive(self): self.assertListEqual( Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'fgCo2'), [FgCo2(None)]) + def test_get_fix_cordex(self): + self.assertListEqual( + Fix.get_fixes( + 'CORDEX', + 'CNRM-ALADIN63', + 'Amon', + 'tas', + extra_facets={'driver':'CNRM-CERFACS-CNRM-CM5'}), + [Tas(None)]) + def test_get_fixes_with_replace(self): self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4'), [Ch4(None)]) From f31d762b87ca67ef8d963a5afe33ea071b37f905 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 21 Oct 2022 16:59:19 +0200 Subject: [PATCH 34/75] Fix flake --- tests/integration/cmor/_fixes/test_fix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cmor/_fixes/test_fix.py b/tests/integration/cmor/_fixes/test_fix.py index 69cb446c09..79f06e899f 100644 --- a/tests/integration/cmor/_fixes/test_fix.py +++ b/tests/integration/cmor/_fixes/test_fix.py @@ -41,7 +41,7 @@ def test_get_fix_cordex(self): 'CNRM-ALADIN63', 'Amon', 'tas', - extra_facets={'driver':'CNRM-CERFACS-CNRM-CM5'}), + extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}), [Tas(None)]) def test_get_fixes_with_replace(self): From 140a681c5516a39e11f2e56be3115c61968278e0 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Sun, 23 Oct 2022 22:49:44 +0200 Subject: [PATCH 35/75] Add tests --- .../cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py | 33 +----- .../cmor/_fixes/cordex/__init__.py | 0 .../cordex/test_cnrm_cerfacs_cnrm_cm5.py | 61 ++++++++++ .../cmor/_fixes/cordex/test_cordex_fixes.py | 96 ++++++++++++++++ .../cmor/_fixes/cordex/test_ichec_ec_earth.py | 46 ++++++++ .../cmor/_fixes/cordex/test_miroc_miroc5.py | 25 +++++ .../_fixes/cordex/test_mohc_hadgem2_es.py | 106 ++++++++++++++++++ .../_fixes/cordex/test_mpi_m_mpi_esm_lr.py | 36 ++++++ .../cmor/_fixes/cordex/test_ncc_noresm1_m.py | 46 ++++++++ 9 files changed, 422 insertions(+), 27 deletions(-) create mode 100644 tests/integration/cmor/_fixes/cordex/__init__.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_cnrm_cerfacs_cnrm_cm5.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_ichec_ec_earth.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_mpi_m_mpi_esm_lr.py create mode 100644 tests/integration/cmor/_fixes/cordex/test_ncc_noresm1_m.py diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py index dfdc7153e3..af0348767a 100644 --- a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py @@ -1,16 +1,16 @@ """Fixes for rcm CNRM-ALADIN63 driven by CNRM-CERFACS-CNRM-CM5.""" -from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.shared import add_scalar_height_coord - import numpy as np +from esmvalcore.cmor._fixes.cordex.cordex_fixes import TimeLongName as BaseFix +from esmvalcore.cmor._fixes.shared import add_scalar_height_coord +from esmvalcore.cmor.fix import Fix + class Tas(Fix): """Fixes for tas.""" def fix_metadata(self, cubes): - """ - Add height (2m) coordinate and correct long_name for time. + """Add height (2m) coordinate and correct long_name for time. Parameters ---------- @@ -20,7 +20,6 @@ def fix_metadata(self, cubes): Returns ------- iris.cube.CubeList - """ for cube in cubes: add_scalar_height_coord(cube) @@ -31,24 +30,4 @@ def fix_metadata(self, cubes): return cubes -class Pr(Fix): - """Fixes for pr.""" - - def fix_metadata(self, cubes): - """ - Correct long_name for time. - - Parameters - ---------- - cubes : iris.cube.CubeList - Input cubes. - - Returns - ------- - iris.cube.CubeList - - """ - for cube in cubes: - cube.coord('time').long_name = 'time' - - return cubes +Pr = BaseFix diff --git a/tests/integration/cmor/_fixes/cordex/__init__.py b/tests/integration/cmor/_fixes/cordex/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/cmor/_fixes/cordex/test_cnrm_cerfacs_cnrm_cm5.py b/tests/integration/cmor/_fixes/cordex/test_cnrm_cerfacs_cnrm_cm5.py new file mode 100644 index 0000000000..2b29c7185c --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_cnrm_cerfacs_cnrm_cm5.py @@ -0,0 +1,61 @@ +"""Tests for the fixes for driver CNRM-CERFACS-CNRM-CM5.""" +import iris +import pytest + +from esmvalcore.cmor._fixes.cordex.cnrm_cerfacs_cnrm_cm5 import cnrm_aladin63 +from esmvalcore.cmor.fix import Fix + + +@pytest.fixture +def cubes(): + correct_time_coord = iris.coords.DimCoord([0.0], + var_name='time', + standard_name='time', + long_name='time') + correct_height_coord = iris.coords.AuxCoord([2.0], + var_name='height') + wrong_height_coord = iris.coords.AuxCoord([10.0], + var_name='height') + correct_cube = iris.cube.Cube( + [10.0], + var_name='tas', + dim_coords_and_dims=[(correct_time_coord, 0)], + aux_coords_and_dims=[(correct_height_coord, ())] + ) + wrong_cube = iris.cube.Cube( + [10.0], + var_name='tas', + dim_coords_and_dims=[(correct_time_coord, 0)], + aux_coords_and_dims=[(wrong_height_coord, ())] + ) + return iris.cube.CubeList([correct_cube, wrong_cube]) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_mohc_hadrem3ga705_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'MOHC-HadREM3-GA7-05', + 'Amon', + short_name, + extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_cnrm_aladin63_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'CNRM-ALADIN63', + 'Amon', + short_name, + extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}) + assert isinstance(fix[0], Fix) + + +def test_cnrm_aladin63_height_fix(cubes): + fix = cnrm_aladin63.Tas(None) + out_cubes = fix.fix_metadata(cubes) + assert cubes is out_cubes + for cube in out_cubes: + assert cube.coord('height').points == 2.0 diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py new file mode 100644 index 0000000000..d5f719841b --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -0,0 +1,96 @@ +"""Tests for general CORDEX fixes.""" +import iris +import numpy as np +import pytest +from cf_units import Unit + +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + CLMcomCCLM4817, + MOHCHadREM3GA705, + TimeLongName, +) + + +@pytest.fixture +def cubes(): + correct_time_coord = iris.coords.DimCoord([0.0], + var_name='time', + standard_name='time', + long_name='time') + wrong_time_coord = iris.coords.DimCoord([0.0], + var_name='time', + standard_name='time', + long_name='wrong') + correct_lat_coord = iris.coords.DimCoord([0.0], + var_name='lat', + standard_name='latitude', + long_name='latitude') + wrong_lat_coord = iris.coords.DimCoord([0.0], + var_name='latitudeCoord', + standard_name='latitude', + long_name='latitude') + correct_lon_coord = iris.coords.DimCoord([0.0], + var_name='lon', + standard_name='longitude', + long_name='longitude') + wrong_lon_coord = iris.coords.DimCoord([0.0], + var_name='longitudeCoord', + standard_name='longitude', + long_name='longitude') + correct_cube = iris.cube.Cube( + [[[10.0]]], + var_name='tas', + dim_coords_and_dims=[ + (correct_time_coord, 0), + (correct_lat_coord, 1), + (correct_lon_coord, 2)], + ) + wrong_cube = iris.cube.Cube( + [[[10.0]]], + var_name='tas', + dim_coords_and_dims=[ + (wrong_time_coord, 0), + (wrong_lat_coord, 1), + (wrong_lon_coord, 2)], + ) + return iris.cube.CubeList([correct_cube, wrong_cube]) + + +def test_mohchadrem3ga705_fix_metadata(cubes): + fix = MOHCHadREM3GA705(None) + out_cubes = fix.fix_metadata(cubes) + assert cubes is out_cubes + for cube in out_cubes: + for coord in cube.coords(): + right_coord = cubes[0].coord(coord) + assert coord == right_coord + + +def test_timelongname_fix_metadata(cubes): + fix = TimeLongName(None) + out_cubes = fix.fix_metadata(cubes) + assert cubes is out_cubes + for cube in out_cubes: + assert cube.coord('time').long_name == 'time' + + +def test_clmcomcclm4817_fix_metadata(cubes): + cubes[0].coord('time').units = Unit( + 'days since 1850-1-1 00:00:00', + calendar='proleptic_gregorian') + cubes[1].coord('time').units = Unit( + 'days since 1850-1-1 00:00:00', + calendar='gregorian') + for coord in cubes[1].coords(): + coord.points = coord.core_points().astype( + '>f8', casting='same_kind') + + fix = CLMcomCCLM4817(None) + out_cubes = fix.fix_metadata(cubes) + assert cubes is out_cubes + for cube in out_cubes: + assert cube.coord('time').units == Unit( + 'days since 1850-1-1 00:00:00', + calendar='proleptic_gregorian') + for coord in cube.coords(): + assert coord.points.dtype == np.float64 diff --git a/tests/integration/cmor/_fixes/cordex/test_ichec_ec_earth.py b/tests/integration/cmor/_fixes/cordex/test_ichec_ec_earth.py new file mode 100644 index 0000000000..09c345ba01 --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_ichec_ec_earth.py @@ -0,0 +1,46 @@ +"""Tests for the fixes for driver ICHEC-EC-Earth.""" +import pytest + +from esmvalcore.cmor.fix import Fix + + +def test_get_gerics_remo2015_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'GERICS-REMO2015', + 'Amon', + 'pr', + extra_facets={'driver': 'ICHEC-EC-Earth'}) + assert isinstance(fix[0], Fix) + + +def test_get_knmi_racmo22e_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'KNMI-RACMO22E', + 'Amon', + 'pr', + extra_facets={'driver': 'ICHEC-EC-Earth'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_mohc_hadrem3ga705_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'MOHC-HadREM3-GA7-05', + 'Amon', + short_name, + extra_facets={'driver': 'ICHEC-EC-Earth'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_smhi_rca4_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'SMHI-RCA4', + 'Amon', + short_name, + extra_facets={'driver': 'ICHEC-EC-Earth'}) + assert isinstance(fix[0], Fix) diff --git a/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py b/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py new file mode 100644 index 0000000000..058b11fb34 --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py @@ -0,0 +1,25 @@ +"""Tests for the fixes of AWI-CM-1-1-MR.""" +import pytest + +from esmvalcore.cmor.fix import Fix + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_clmcom_cclm4_8_17fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'CLMCOM-CCLM4-8-17', + 'Amon', + short_name, + extra_facets={'driver': 'MIROC-MIROC5'}) + assert isinstance(fix[0], Fix) + + +def test_get_gerics_remo2015_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'GERICS-REMO2015', + 'Amon', + 'pr', + extra_facets={'driver': 'MIROC-MIROC5'}) + assert isinstance(fix[0], Fix) diff --git a/tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py b/tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py new file mode 100644 index 0000000000..2d00b7bb51 --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py @@ -0,0 +1,106 @@ +"""Tests for the fixes for driver MOHC-HadGEM2-ES.""" +import iris +import pytest + +from esmvalcore.cmor._fixes.cordex.mohc_hadgem2_es import dmi_hirham5 +from esmvalcore.cmor.fix import Fix + + +@pytest.fixture +def cubes(): + correct_time_coord = iris.coords.DimCoord([0.0], + var_name='time', + standard_name='time', + long_name='time') + wrong_time_coord = iris.coords.DimCoord([0.0], + var_name='time', + standard_name='time', + long_name='wrong') + correct_lat_coord = iris.coords.DimCoord([0.0], + var_name='lat', + standard_name='latitude', + long_name='latitude') + wrong_lat_coord = iris.coords.DimCoord([0.0], + var_name='latitudeCoord', + standard_name='latitude', + long_name='latitude', + attributes={'wrong': 'attr'}) + correct_lon_coord = iris.coords.DimCoord([0.0], + var_name='lon', + standard_name='longitude', + long_name='longitude') + wrong_lon_coord = iris.coords.DimCoord([0.0], + var_name='longitudeCoord', + standard_name='longitude', + long_name='longitude', + attributes={'wrong': 'attr'}) + correct_cube = iris.cube.Cube( + [[[10.0]]], + var_name='tas', + dim_coords_and_dims=[ + (correct_time_coord, 0), + (correct_lat_coord, 1), + (correct_lon_coord, 2)], + ) + wrong_cube = iris.cube.Cube( + [[[10.0]]], + var_name='tas', + dim_coords_and_dims=[ + (wrong_time_coord, 0), + (wrong_lat_coord, 1), + (wrong_lon_coord, 2)], + ) + return iris.cube.CubeList([correct_cube, wrong_cube]) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_dmi_hirham5_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'DMI-HIRHAM5', + 'Amon', + short_name, + extra_facets={'driver': 'MOHC-HadGEM2-ES'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_gerics_remo2015_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'GERICS-REMO2015', + 'Amon', + short_name, + extra_facets={'driver': 'MOHC-HadGEM2-ES'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_mohc_hadrem3ga705_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'MOHC-HadREM3-GA7-05', + 'Amon', + short_name, + extra_facets={'driver': 'MOHC-HadGEM2-ES'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_smhi_rca4_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'SMHI-RCA4', + 'Amon', + short_name, + extra_facets={'driver': 'MOHC-HadGEM2-ES'}) + assert isinstance(fix[0], Fix) + + +def test_dmi_hirham5_fix(cubes): + fix = dmi_hirham5.Pr(None) + out_cubes = fix.fix_metadata(cubes) + assert cubes is out_cubes + for cube in out_cubes: + assert cube.coord('latitude').attributes == {} + assert cube.coord('longitude').attributes == {} diff --git a/tests/integration/cmor/_fixes/cordex/test_mpi_m_mpi_esm_lr.py b/tests/integration/cmor/_fixes/cordex/test_mpi_m_mpi_esm_lr.py new file mode 100644 index 0000000000..30c813297f --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_mpi_m_mpi_esm_lr.py @@ -0,0 +1,36 @@ +"""Tests for the fixes of driver MPI-M-MPI-ESM-LR.""" +import pytest + +from esmvalcore.cmor.fix import Fix + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_ictp_regcm4_6_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'ICTP-REGCM4-6', + 'Amon', + short_name, + extra_facets={'driver': 'MPI-M-MPI-ESM-LR'}) + assert isinstance(fix[0], Fix) + + +def test_get_knmi_racmo22e_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'KNMI-RACMO22E', + 'Amon', + 'pr', + extra_facets={'driver': 'MPI-M-MPI-ESM-LR'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_mohc_hadrem3ga705_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'MOHC-HadREM3-GA7-05', + 'Amon', + short_name, + extra_facets={'driver': 'MPI-M-MPI-ESM-LR'}) + assert isinstance(fix[0], Fix) diff --git a/tests/integration/cmor/_fixes/cordex/test_ncc_noresm1_m.py b/tests/integration/cmor/_fixes/cordex/test_ncc_noresm1_m.py new file mode 100644 index 0000000000..e071c8d7ea --- /dev/null +++ b/tests/integration/cmor/_fixes/cordex/test_ncc_noresm1_m.py @@ -0,0 +1,46 @@ +"""Tests for the fixes of driver NCC-NorESM1-M.""" +import pytest + +from esmvalcore.cmor.fix import Fix + + +def test_get_gerics_remo2015_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'GERICS-REMO2015', + 'Amon', + 'pr', + extra_facets={'driver': 'NCC-NorESM1-M'}) + assert isinstance(fix[0], Fix) + + +def test_get_knmi_racmo22e_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'KNMI-RACMO22E', + 'Amon', + 'pr', + extra_facets={'driver': 'NCC-NorESM1-M'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_mohc_hadrem3ga705_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'MOHC-HadREM3-GA7-05', + 'Amon', + short_name, + extra_facets={'driver': 'NCC-NorESM1-M'}) + assert isinstance(fix[0], Fix) + + +@pytest.mark.parametrize('short_name', ['pr', 'tas']) +def test_get_smhi_rca4_fix(short_name): + fix = Fix.get_fixes( + 'CORDEX', + 'SMHI-RCA4', + 'Amon', + short_name, + extra_facets={'driver': 'NCC-NorESM1-M'}) + assert isinstance(fix[0], Fix) From 052a575d6c62b481c24596c953b1d9d364da12eb Mon Sep 17 00:00:00 2001 From: sloosvel Date: Sun, 23 Oct 2022 22:54:33 +0200 Subject: [PATCH 36/75] Fix test --- tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py b/tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py index 2d00b7bb51..408fe4d5a6 100644 --- a/tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py +++ b/tests/integration/cmor/_fixes/cordex/test_mohc_hadgem2_es.py @@ -53,13 +53,12 @@ def cubes(): return iris.cube.CubeList([correct_cube, wrong_cube]) -@pytest.mark.parametrize('short_name', ['pr', 'tas']) -def test_get_dmi_hirham5_fix(short_name): +def test_get_dmi_hirham5_fix(): fix = Fix.get_fixes( 'CORDEX', 'DMI-HIRHAM5', 'Amon', - short_name, + 'pr', extra_facets={'driver': 'MOHC-HadGEM2-ES'}) assert isinstance(fix[0], Fix) From 4d2c85d5aa4c5fc15cfadcb0a5c4a4ecef8ab24c Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 24 Oct 2022 09:47:06 +0200 Subject: [PATCH 37/75] Add test for bounds --- .../cmor/_fixes/cordex/test_cordex_fixes.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index d5f719841b..bd3a84381e 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -21,11 +21,11 @@ def cubes(): var_name='time', standard_name='time', long_name='wrong') - correct_lat_coord = iris.coords.DimCoord([0.0], + correct_lat_coord = iris.coords.DimCoord([0.0, 1.0], var_name='lat', standard_name='latitude', long_name='latitude') - wrong_lat_coord = iris.coords.DimCoord([0.0], + wrong_lat_coord = iris.coords.DimCoord([0.0, 1.0], var_name='latitudeCoord', standard_name='latitude', long_name='latitude') @@ -38,7 +38,7 @@ def cubes(): standard_name='longitude', long_name='longitude') correct_cube = iris.cube.Cube( - [[[10.0]]], + [[[10.0], [10.0]]], var_name='tas', dim_coords_and_dims=[ (correct_time_coord, 0), @@ -46,7 +46,7 @@ def cubes(): (correct_lon_coord, 2)], ) wrong_cube = iris.cube.Cube( - [[[10.0]]], + [[[10.0], [10.0]]], var_name='tas', dim_coords_and_dims=[ (wrong_time_coord, 0), @@ -84,6 +84,10 @@ def test_clmcomcclm4817_fix_metadata(cubes): for coord in cubes[1].coords(): coord.points = coord.core_points().astype( '>f8', casting='same_kind') + lat = cubes[1].coord('latitude') + lat.guess_bounds() + lat.bounds = lat.core_bounds().astype( + '>f4', casting='same_kind') fix = CLMcomCCLM4817(None) out_cubes = fix.fix_metadata(cubes) From a8f28ff1fa31e60e84cf3f091df2b89dcecadfba Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 24 Oct 2022 09:49:46 +0200 Subject: [PATCH 38/75] Fix codacy --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 4 +--- esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 75f8483d6e..769d95d0b0 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -9,9 +9,7 @@ class MOHCHadREM3GA705(Fix): """General fix for MOHC-HadREM3-GA7-05.""" def fix_metadata(self, cubes): - """ - Fix time long_name. - Fix latitude and longitude var_name. + """Fix time long_name, and latitude and longitude var_name. Parameters ---------- diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py index 6c1804b9d8..fbf5633d13 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py @@ -6,8 +6,7 @@ class Pr(Fix): """Fixes for pr.""" def fix_metadata(self, cubes): - """ - Remove latitude and longitude attributes + """Remove latitude and longitude attributes. Parameters ---------- From f8bbeedd723d2578c46a6ec43e102f4fea439d8a Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 24 Oct 2022 09:58:00 +0200 Subject: [PATCH 39/75] Fix docstring --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 769d95d0b0..06fcf47c32 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -56,7 +56,8 @@ class CLMcomCCLM4817(Fix): """Fixes for CLMcom-CCLM4-8-17.""" def fix_metadata(self, cubes): - """ + """ Fix calendars. + Set calendar to 'proleptic_gregorian' to avoid concatenation issues between historical and scenario runs. From 7899214ab6f92c13ad7b8bf3ce63b2df8fcd4483 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 24 Oct 2022 10:35:29 +0200 Subject: [PATCH 40/75] Remove whitespace --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 06fcf47c32..54f9ca1415 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -56,7 +56,7 @@ class CLMcomCCLM4817(Fix): """Fixes for CLMcom-CCLM4-8-17.""" def fix_metadata(self, cubes): - """ Fix calendars. + """Fix calendars. Set calendar to 'proleptic_gregorian' to avoid concatenation issues between historical and From 7132ac5481730016a67d7ac79869f59fba52cdf5 Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Mon, 21 Nov 2022 12:02:37 +0100 Subject: [PATCH 41/75] Apply suggestions from code review Co-authored-by: Pep Cos <66776374+pepcos@users.noreply.github.com> --- .../cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py | 2 +- esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py index a2b791ceaa..004cbc412f 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py @@ -1,4 +1,4 @@ -"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by MOHC-HadGEM2.""" +"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by MOHC-HadGEM2-ES.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( MOHCHadREM3GA705 as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py index 8108e238b8..543df3670a 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py @@ -1,4 +1,4 @@ -"""Fixes for rcm SMHI-RCA4 driven by MOHC-HadGEM2.""" +"""Fixes for rcm SMHI-RCA4 driven by MOHC-HadGEM2-ES.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( TimeLongName as BaseFix) From 1ac59aec3688926191feae7317b57c51bc18f7cc Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 2 Dec 2022 20:18:18 +0100 Subject: [PATCH 42/75] First attempt at boundary fix --- environment.yml | 1 + esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 68 ++++++++++++++++--- setup.py | 1 + 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/environment.yml b/environment.yml index 21795e34b2..3c8e672f12 100644 --- a/environment.yml +++ b/environment.yml @@ -22,4 +22,5 @@ dependencies: - pip!=21.3 - python>=3.8 - python-stratify + - py-cordex - scipy>=1.6 diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 54f9ca1415..174f794bf6 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -1,6 +1,7 @@ """Fixes that are shared between datasets and drivers.""" from cf_units import Unit -import numpy as np +import cordex as cx +import iris from esmvalcore.cmor.fix import Fix @@ -80,13 +81,60 @@ def fix_metadata(self, cubes): 'days since 1850-1-1 00:00:00', calendar='proleptic_gregorian' ) - for coord in cube.coords(): - if coord.dtype in ['>f8', '>f4']: - coord.points = coord.core_points().astype( - np.float64, casting='same_kind') - if coord.bounds is not None: - coord.bounds = coord.core_bounds().astype( - np.float64, casting='same_kind') - # Further issues appear, maybe because historical data lat/lon - # does not have bounds whereas scenario data has them. return cubes + +class AllVars(Fix): + """General CORDEX grid fix""" + def _fix_rotated_coords(self, cube): + data_domain = self.extra_facets['domain'] + domain = cx.cordex_domain(data_domain, add_vertices=True) + domain_info = cx.domain_info(data_domain) + for dim_coord in ['rlat', 'rlon']: + old_coord = cube.coord(var_name=dim_coord) + old_coord_dims = old_coord.cube_dims(cube) + points = domain[dim_coord].data + coord_system = iris.coord_systems.RotatedGeogCS( + grid_north_pole_latitude=domain_info['pollat'], + grid_north_pole_longitude=domain_info['pollon'] + ) + new_coord = iris.coords.DimCoord( + points, + var_name=dim_coord, + standard_name=domain[dim_coord].standard_name, + long_name=domain[dim_coord].long_name, + units=Unit('degrees'), + coord_system=coord_system, + ) + new_coord.guess_bounds() + cube.remove_coord(old_coord) + cube.add_dim_coord(new_coord, old_coord_dims) + + def _fix_geographical_coords(self, cube): + data_domain = self.extra_facets['domain'] + domain = cx.cordex_domain(data_domain, add_vertices=True) + for aux_coord in ['lat', 'lon']: + # check if la coord existeix + old_coord = cube.coord(var_name=aux_coord) + points = domain[aux_coord].data + bounds = domain[f'{aux_coord}_vertices'].data + new_coord = iris.coords.AuxCoord( + points, + var_name=aux_coord, + standard_name=domain[aux_coord].standard_name, + long_name=domain[aux_coord].long_name, + units=Unit(domain[aux_coord].units), + bounds=bounds + ) + cube.remove_coord(old_coord) + cube.add_aux_coord(new_coord, (1,2)) + + + def fix_metadata(self, cubes): + for cube in cubes: + coord_system = cube.coord(axis='X').coord_system + if isinstance(coord_system, iris.coord_systems.RotatedGeogCS): + self._fix_rotated_coords(cube) + self._fix_geographical_coords(cube) + + return cubes + diff --git a/setup.py b/setup.py index 25e996e274..59e439fcb4 100755 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ 'psutil', 'pybtex', 'pyyaml', + 'py-cordex', 'requests', 'scipy>=1.6', 'scitools-iris>=3.2.1', From 683ca8d8ff11e98dbbfb9839c86c6a9660fd0cfe Mon Sep 17 00:00:00 2001 From: sloosvel Date: Sat, 3 Dec 2022 23:21:31 +0100 Subject: [PATCH 43/75] Add general grid fix for rotated datasets --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 15 ++++----- esmvalcore/cmor/_fixes/fix.py | 32 ++++++++++++------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 174f794bf6..2a1c833a1b 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -4,6 +4,7 @@ import iris from esmvalcore.cmor.fix import Fix +from iris.coord_systems import RotatedGeogCS class MOHCHadREM3GA705(Fix): @@ -90,7 +91,7 @@ def _fix_rotated_coords(self, cube): domain = cx.cordex_domain(data_domain, add_vertices=True) domain_info = cx.domain_info(data_domain) for dim_coord in ['rlat', 'rlon']: - old_coord = cube.coord(var_name=dim_coord) + old_coord = cube.coord(domain[dim_coord].standard_name) old_coord_dims = old_coord.cube_dims(cube) points = domain[dim_coord].data coord_system = iris.coord_systems.RotatedGeogCS( @@ -113,8 +114,8 @@ def _fix_geographical_coords(self, cube): data_domain = self.extra_facets['domain'] domain = cx.cordex_domain(data_domain, add_vertices=True) for aux_coord in ['lat', 'lon']: - # check if la coord existeix - old_coord = cube.coord(var_name=aux_coord) + old_coord = cube.coord(domain[aux_coord].standard_name) + cube.remove_coord(old_coord) points = domain[aux_coord].data bounds = domain[f'{aux_coord}_vertices'].data new_coord = iris.coords.AuxCoord( @@ -125,14 +126,12 @@ def _fix_geographical_coords(self, cube): units=Unit(domain[aux_coord].units), bounds=bounds ) - cube.remove_coord(old_coord) - cube.add_aux_coord(new_coord, (1,2)) - + cube.add_aux_coord(new_coord, (1, 2)) def fix_metadata(self, cubes): for cube in cubes: - coord_system = cube.coord(axis='X').coord_system - if isinstance(coord_system, iris.coord_systems.RotatedGeogCS): + coord_system = cube.coord_system() + if isinstance(coord_system, RotatedGeogCS): self._fix_rotated_coords(cube) self._fix_geographical_coords(cube) diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 7e473c36ba..646c188f32 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -157,15 +157,26 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): extra_facets = {} fixes = [] - try: - if project == 'cordex': - driver = extra_facets['driver'].replace('-', '_').lower() - fixes_module = importlib.import_module( - f'esmvalcore.cmor._fixes.{project}.{driver}.{dataset}') - else: - fixes_module = importlib.import_module( - 'esmvalcore.cmor._fixes.{0}.{1}'.format(project, dataset)) - + + if project == 'cordex': + driver = extra_facets['driver'].replace('-', '_').lower() + fixes_modules = [] + try: + fixes_modules.append(importlib.import_module( + f'esmvalcore.cmor._fixes.{project}.{driver}.{dataset}' + )) + except ImportError: + pass + fixes_modules.append(importlib.import_module( + f'esmvalcore.cmor._fixes.cordex.cordex_fixes')) + else: + try: + fixes_modules = [importlib.import_module( + f'esmvalcore.cmor._fixes.{project}.{dataset}')] + except ImportError: + pass + + for fixes_module in fixes_modules: classes = inspect.getmembers(fixes_module, inspect.isclass) classes = dict((name.lower(), value) for name, value in classes) for fix_name in (short_name, mip.lower(), 'allvars'): @@ -173,8 +184,7 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): fixes.append(classes[fix_name](vardef, extra_facets)) except KeyError: pass - except ImportError: - pass + return fixes @staticmethod From c63e76af75720a65ed7c2efd86957bbc06804e7c Mon Sep 17 00:00:00 2001 From: sloosvel Date: Sat, 3 Dec 2022 23:22:23 +0100 Subject: [PATCH 44/75] Add tas fix for UHOH-WRF361H --- .../_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py new file mode 100644 index 0000000000..69b0e6f63c --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py @@ -0,0 +1,17 @@ +"""Fixes for rcm UHOH-WRF361H driven by MIROC-MIROC5.""" +import iris +from esmvalcore.cmor.fix import Fix + +class Tas(Fix): + def fix_metadata(self, cubes): + fixed_cubes = iris.cube.CubeList() + for cube in cubes: + height = cube.coord('height') + if isinstance(height, iris.coords.DimCoord): + iris.util.demote_dim_coord_to_aux_coord( + cube, + height + ) + fixed_cubes.append(iris.util.squeeze(cube)) + return fixed_cubes + From 39b8198b449c004ee4e9b1d4bcc1e2a74a0a0515 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 5 Dec 2022 09:30:57 +0100 Subject: [PATCH 45/75] Fix flake --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 9 +++++---- .../cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py | 2 +- esmvalcore/cmor/_fixes/fix.py | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 2a1c833a1b..5ba222883a 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -83,7 +83,8 @@ def fix_metadata(self, cubes): calendar='proleptic_gregorian' ) return cubes - + + class AllVars(Fix): """General CORDEX grid fix""" def _fix_rotated_coords(self, cube): @@ -127,13 +128,13 @@ def _fix_geographical_coords(self, cube): bounds=bounds ) cube.add_aux_coord(new_coord, (1, 2)) - + def fix_metadata(self, cubes): for cube in cubes: coord_system = cube.coord_system() if isinstance(coord_system, RotatedGeogCS): self._fix_rotated_coords(cube) self._fix_geographical_coords(cube) - + return cubes - + diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py index 69b0e6f63c..ec73325005 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py @@ -2,6 +2,7 @@ import iris from esmvalcore.cmor.fix import Fix + class Tas(Fix): def fix_metadata(self, cubes): fixed_cubes = iris.cube.CubeList() @@ -14,4 +15,3 @@ def fix_metadata(self, cubes): ) fixed_cubes.append(iris.util.squeeze(cube)) return fixed_cubes - diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 646c188f32..51af5ec425 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -157,7 +157,7 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): extra_facets = {} fixes = [] - + if project == 'cordex': driver = extra_facets['driver'].replace('-', '_').lower() fixes_modules = [] @@ -168,14 +168,14 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): except ImportError: pass fixes_modules.append(importlib.import_module( - f'esmvalcore.cmor._fixes.cordex.cordex_fixes')) + 'esmvalcore.cmor._fixes.cordex.cordex_fixes')) else: try: fixes_modules = [importlib.import_module( f'esmvalcore.cmor._fixes.{project}.{dataset}')] except ImportError: pass - + for fixes_module in fixes_modules: classes = inspect.getmembers(fixes_module, inspect.isclass) classes = dict((name.lower(), value) for name, value in classes) @@ -184,7 +184,7 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): fixes.append(classes[fix_name](vardef, extra_facets)) except KeyError: pass - + return fixes @staticmethod From a3560f450b2a83748dbad129418277611d257e29 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 5 Dec 2022 09:34:10 +0100 Subject: [PATCH 46/75] Remove blank line --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 5ba222883a..fbcdd20f36 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -137,4 +137,3 @@ def fix_metadata(self, cubes): self._fix_geographical_coords(cube) return cubes - From 603db38ca696cbe41c085df6cae234621a27173e Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 5 Dec 2022 09:56:02 +0100 Subject: [PATCH 47/75] Fix tests --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 8 ++++++++ esmvalcore/cmor/_fixes/fix.py | 6 +++--- tests/integration/cmor/_fixes/test_fix.py | 3 ++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index fbcdd20f36..f2ff24b674 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -1,6 +1,7 @@ """Fixes that are shared between datasets and drivers.""" from cf_units import Unit import cordex as cx +import numpy as np import iris from esmvalcore.cmor.fix import Fix @@ -82,6 +83,13 @@ def fix_metadata(self, cubes): 'days since 1850-1-1 00:00:00', calendar='proleptic_gregorian' ) + for coord in cube.coords(): + if coord.dtype in ['>f8', '>f4']: + coord.points = coord.core_points().astype( + np.float64, casting='same_kind') + if coord.bounds is not None: + coord.bounds = coord.core_bounds().astype( + np.float64, casting='same_kind') return cubes diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 51af5ec425..d3766007d8 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -158,9 +158,9 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): fixes = [] + fixes_modules = [] if project == 'cordex': driver = extra_facets['driver'].replace('-', '_').lower() - fixes_modules = [] try: fixes_modules.append(importlib.import_module( f'esmvalcore.cmor._fixes.{project}.{driver}.{dataset}' @@ -171,8 +171,8 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): 'esmvalcore.cmor._fixes.cordex.cordex_fixes')) else: try: - fixes_modules = [importlib.import_module( - f'esmvalcore.cmor._fixes.{project}.{dataset}')] + fixes_modules.append(importlib.import_module( + f'esmvalcore.cmor._fixes.{project}.{dataset}')) except ImportError: pass diff --git a/tests/integration/cmor/_fixes/test_fix.py b/tests/integration/cmor/_fixes/test_fix.py index 79f06e899f..874959a864 100644 --- a/tests/integration/cmor/_fixes/test_fix.py +++ b/tests/integration/cmor/_fixes/test_fix.py @@ -14,6 +14,7 @@ from esmvalcore.cmor._fixes.cmip6.cesm2 import Omon, Tos from esmvalcore.cmor._fixes.cordex.cnrm_cerfacs_cnrm_cm5.cnrm_aladin63 import ( Tas) +from esmvalcore.cmor._fixes.cordex.cordex_fixes import AllVars from esmvalcore.cmor.fix import Fix @@ -42,7 +43,7 @@ def test_get_fix_cordex(self): 'Amon', 'tas', extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}), - [Tas(None)]) + [Tas(None), AllVars(None)]) def test_get_fixes_with_replace(self): self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4'), From d394c648cb267beba98d25ac50622d869a03b003 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 5 Dec 2022 11:59:17 +0100 Subject: [PATCH 48/75] Update calendar names --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 2 +- tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index f2ff24b674..8c240506ab 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -78,7 +78,7 @@ def fix_metadata(self, cubes): """ for cube in cubes: - if cube.coord('time').units.calendar == 'gregorian': + if cube.coord('time').units.calendar == 'standard': cube.coord('time').units = Unit( 'days since 1850-1-1 00:00:00', calendar='proleptic_gregorian' diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index bd3a84381e..a586a0fefe 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -80,7 +80,7 @@ def test_clmcomcclm4817_fix_metadata(cubes): calendar='proleptic_gregorian') cubes[1].coord('time').units = Unit( 'days since 1850-1-1 00:00:00', - calendar='gregorian') + calendar='standard') for coord in cubes[1].coords(): coord.points = coord.core_points().astype( '>f8', casting='same_kind') From 78f45c3487ea4c0737ef3a77fd8b6714e4c7409d Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 5 Dec 2022 17:13:20 +0100 Subject: [PATCH 49/75] Add grid fix test --- .../cmor/_fixes/cordex/test_cordex_fixes.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index a586a0fefe..e44585e8e9 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -1,10 +1,12 @@ """Tests for general CORDEX fixes.""" +import cordex as cx import iris import numpy as np import pytest from cf_units import Unit from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + AllVars, CLMcomCCLM4817, MOHCHadREM3GA705, TimeLongName, @@ -56,6 +58,49 @@ def cubes(): return iris.cube.CubeList([correct_cube, wrong_cube]) +@pytest.fixture +def cordex_cubes(): + coord_system = iris.coord_systems.RotatedGeogCS( + grid_north_pole_latitude=39.25, + grid_north_pole_longitude=-162 + ) + time = iris.coords.DimCoord(np.arange(0, 3), + var_name='time', + standard_name='time') + + rlat = iris.coords.DimCoord(np.arange(0, 412), + var_name='rlat', + standard_name='grid_latitude', + coord_system=coord_system, + ) + rlon = iris.coords.DimCoord(np.arange(0, 424), + var_name='rlon', + standard_name='grid_longitude', + coord_system=coord_system, + ) + lat = iris.coords.AuxCoord(np.ones((412, 424)), + var_name='lat', + standard_name='latitude') + lon = iris.coords.AuxCoord(np.ones((412, 424)), + var_name='lon', + standard_name='longitude') + + cube = iris.cube.Cube( + np.ones((3, 412, 424)), + var_name='tas', + dim_coords_and_dims=[ + (time, 0), + (rlat, 1), + (rlon, 2)], + aux_coords_and_dims=[ + (lat, (1,2)), + (lon, (1,2)) + ] + + ) + return iris.cube.CubeList([cube]) + + def test_mohchadrem3ga705_fix_metadata(cubes): fix = MOHCHadREM3GA705(None) out_cubes = fix.fix_metadata(cubes) @@ -98,3 +143,18 @@ def test_clmcomcclm4817_fix_metadata(cubes): calendar='proleptic_gregorian') for coord in cube.coords(): assert coord.points.dtype == np.float64 + + +def test_rotated_grid_fix(cordex_ + fix = AllVars( + vardef=None, + extra_facets={'domain': 'EUR-11'}) + out_cubes = fix.fix_metadata(cordex_cubes) + domain = cx.cordex_domain('EUR-11', add_vertices=True) + assert cordex_cubes is out_cubes + for cube in out_cubes: + for coord in ['rlat', 'rlon', 'lat', 'lon']: + cube_coord = cube.coord(var_name=coord) + domain_coord = domain[coord].data + np.testing.assert_array_equal( + cube_coord.points, domain_coord) From 60901f83000a670a91c84e9707f3a3f08459b20a Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 5 Dec 2022 17:26:55 +0100 Subject: [PATCH 50/75] Fix flake --- tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index e44585e8e9..2af866d047 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -145,10 +145,11 @@ def test_clmcomcclm4817_fix_metadata(cubes): assert coord.points.dtype == np.float64 -def test_rotated_grid_fix(cordex_ +def test_rotated_grid_fix(cordex_cubes): fix = AllVars( vardef=None, extra_facets={'domain': 'EUR-11'}) + out_cubes = fix.fix_metadata(cordex_cubes) domain = cx.cordex_domain('EUR-11', add_vertices=True) assert cordex_cubes is out_cubes From 29614b2fba4fa344fb36ce87171ba8709e9669ce Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 5 Dec 2022 17:32:10 +0100 Subject: [PATCH 51/75] Fix flake --- tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index 2af866d047..6416533067 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -84,7 +84,7 @@ def cordex_cubes(): lon = iris.coords.AuxCoord(np.ones((412, 424)), var_name='lon', standard_name='longitude') - + cube = iris.cube.Cube( np.ones((3, 412, 424)), var_name='tas', @@ -93,8 +93,8 @@ def cordex_cubes(): (rlat, 1), (rlon, 2)], aux_coords_and_dims=[ - (lat, (1,2)), - (lon, (1,2)) + (lat, (1, 2)), + (lon, (1, 2)) ] ) From 72638d234154ce4a0cde3df8ff62e668305865dc Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 5 Dec 2022 17:58:58 +0100 Subject: [PATCH 52/75] Add tests for UHOH-WRF361H --- .../cmor/_fixes/cordex/test_miroc_miroc5.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py b/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py index 058b11fb34..c9db25c451 100644 --- a/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py +++ b/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py @@ -1,9 +1,28 @@ """Tests for the fixes of AWI-CM-1-1-MR.""" import pytest +import iris +from esmvalcore.cmor._fixes.cordex.miroc_miroc5 import uhoh_wrf361h from esmvalcore.cmor.fix import Fix +@pytest.fixture +def cubes(): + correct_time_coord = iris.coords.DimCoord([0.0, 1.0], + var_name='time', + standard_name='time', + long_name='time') + wrong_height_coord = iris.coords.DimCoord([2.0], + var_name='height') + wrong_cube = iris.cube.Cube( + [[10.0], [10.0]], + var_name='tas', + dim_coords_and_dims=[ + (correct_time_coord, 0), + (wrong_height_coord, 1)], + ) + return iris.cube.CubeList([wrong_cube]) + @pytest.mark.parametrize('short_name', ['pr', 'tas']) def test_get_clmcom_cclm4_8_17fix(short_name): fix = Fix.get_fixes( @@ -23,3 +42,20 @@ def test_get_gerics_remo2015_fix(): 'pr', extra_facets={'driver': 'MIROC-MIROC5'}) assert isinstance(fix[0], Fix) + + +def test_get_gerics_remo2015_fix(): + fix = Fix.get_fixes( + 'CORDEX', + 'UHOH-WRF361H', + 'Amon', + 'tas', + extra_facets={'driver': 'MIROC-MIROC5'}) + assert isinstance(fix[0], Fix) + + +def test_uhoh_wrf361h_height_fix(cubes): + fix = uhoh_wrf361h.Tas(None) + out_cubes = fix.fix_metadata(cubes) + for cube in out_cubes: + assert cube.ndim == 1 \ No newline at end of file From 0f3a1397ecec274591f395b1c4419efb441387ba Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 5 Dec 2022 18:04:17 +0100 Subject: [PATCH 53/75] Fix tests --- tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py b/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py index c9db25c451..a177498930 100644 --- a/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py +++ b/tests/integration/cmor/_fixes/cordex/test_miroc_miroc5.py @@ -23,6 +23,7 @@ def cubes(): ) return iris.cube.CubeList([wrong_cube]) + @pytest.mark.parametrize('short_name', ['pr', 'tas']) def test_get_clmcom_cclm4_8_17fix(short_name): fix = Fix.get_fixes( @@ -44,7 +45,7 @@ def test_get_gerics_remo2015_fix(): assert isinstance(fix[0], Fix) -def test_get_gerics_remo2015_fix(): +def test_get_uhoh_wrf361h_fix(): fix = Fix.get_fixes( 'CORDEX', 'UHOH-WRF361H', @@ -58,4 +59,4 @@ def test_uhoh_wrf361h_height_fix(cubes): fix = uhoh_wrf361h.Tas(None) out_cubes = fix.fix_metadata(cubes) for cube in out_cubes: - assert cube.ndim == 1 \ No newline at end of file + assert cube.ndim == 1 From ac3fe855be6f34a94a3b2c28f4682d2f72410f80 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 6 Dec 2022 10:24:22 +0100 Subject: [PATCH 54/75] Improve code coverage --- tests/integration/cmor/_fixes/test_fix.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integration/cmor/_fixes/test_fix.py b/tests/integration/cmor/_fixes/test_fix.py index 874959a864..16629549fe 100644 --- a/tests/integration/cmor/_fixes/test_fix.py +++ b/tests/integration/cmor/_fixes/test_fix.py @@ -45,6 +45,16 @@ def test_get_fix_cordex(self): extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}), [Tas(None), AllVars(None)]) + def test_get_grid_fix_cordex(self): + self.assertListEqual( + Fix.get_fixes( + 'CORDEX', + 'CNRM-ALADIN53', + 'Amon', + 'tas', + extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}), + [AllVars(None)]) + def test_get_fixes_with_replace(self): self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4'), [Ch4(None)]) From 5e7fadfaafaae9739fefe6662feb6a5102eba8d3 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Tue, 6 Dec 2022 10:43:31 +0100 Subject: [PATCH 55/75] Add docstrings --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 25 +++++++++++++++++-- .../cordex/miroc_miroc5/uhoh_wrf361h.py | 17 +++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 8c240506ab..ce5d62d414 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -4,8 +4,8 @@ import numpy as np import iris -from esmvalcore.cmor.fix import Fix from iris.coord_systems import RotatedGeogCS +from esmvalcore.cmor.fix import Fix class MOHCHadREM3GA705(Fix): @@ -94,8 +94,10 @@ def fix_metadata(self, cubes): class AllVars(Fix): - """General CORDEX grid fix""" + """General CORDEX grid fix.""" + def _fix_rotated_coords(self, cube): + """Fix rotated coordinates.""" data_domain = self.extra_facets['domain'] domain = cx.cordex_domain(data_domain, add_vertices=True) domain_info = cx.domain_info(data_domain) @@ -120,6 +122,7 @@ def _fix_rotated_coords(self, cube): cube.add_dim_coord(new_coord, old_coord_dims) def _fix_geographical_coords(self, cube): + """Fix geographical coordinates.""" data_domain = self.extra_facets['domain'] domain = cx.cordex_domain(data_domain, add_vertices=True) for aux_coord in ['lat', 'lon']: @@ -138,6 +141,24 @@ def _fix_geographical_coords(self, cube): cube.add_aux_coord(new_coord, (1, 2)) def fix_metadata(self, cubes): + """Fix CORDEX rotated grids. + + Set rotated and geographical coordinates to the + values given by each domain specification. + + The domain specifications are retrieved from the + py-cordex package. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ for cube in cubes: coord_system = cube.coord_system() if isinstance(coord_system, RotatedGeogCS): diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py index ec73325005..3767c653ea 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py @@ -4,7 +4,24 @@ class Tas(Fix): + """Fixes for tas.""" + def fix_metadata(self, cubes): + """Fix tas coordinates. + + Set height as an auxiliary coordinate instead + of as a dimensional coordinate. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ fixed_cubes = iris.cube.CubeList() for cube in cubes: height = cube.coord('height') From ebbaaf82b8fb0a73b6e213b631d2664706a10571 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 14 Dec 2022 10:45:26 +0100 Subject: [PATCH 56/75] Address review comments --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 37 ++++++++++++++++++- .../cmor/_fixes/cordex/test_cordex_fixes.py | 35 +++++++++++++++--- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index ce5d62d414..0abc3eeab7 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -3,10 +3,12 @@ import cordex as cx import numpy as np import iris +import logging from iris.coord_systems import RotatedGeogCS from esmvalcore.cmor.fix import Fix - +from esmvalcore.exceptions import RecipeError +logger = logging.getLogger(__name__) class MOHCHadREM3GA705(Fix): """General fix for MOHC-HadREM3-GA7-05.""" @@ -96,6 +98,28 @@ def fix_metadata(self, cubes): class AllVars(Fix): """General CORDEX grid fix.""" + _grid_diff_msg = ('Maximum difference between original {} ' + 'points and standard {} domain points ' + 'for dataset {} and driver {} is: {}') + + def _check_grid_differences(self, old_coord, new_coord): + """Check differences between coords.""" + diff = np.max(np.abs(old_coord.points - new_coord.points)) + logger.debug( + self._grid_diff_msg, + new_coord.var_name, + self.extra_facets['domain'], + self.extra_facets['dataset'], + self.extra_facets['driver'], + diff + ) + + if diff > 10e-4: + raise RecipeError( + "Differences between the original grid and the " + f"standarised grid are above 10e-4 {new_coord.units}.", + ) + def _fix_rotated_coords(self, cube): """Fix rotated coordinates.""" data_domain = self.extra_facets['domain'] @@ -117,6 +141,7 @@ def _fix_rotated_coords(self, cube): units=Unit('degrees'), coord_system=coord_system, ) + self._check_grid_differences(old_coord, new_coord) new_coord.guess_bounds() cube.remove_coord(old_coord) cube.add_dim_coord(new_coord, old_coord_dims) @@ -138,7 +163,15 @@ def _fix_geographical_coords(self, cube): units=Unit(domain[aux_coord].units), bounds=bounds ) - cube.add_aux_coord(new_coord, (1, 2)) + logger.info(old_coord.units) + self._check_grid_differences(old_coord, new_coord) + aux_coord_dims = ( + cube.coord(var_name='rlat').cube_dims(cube) + + cube.coord(var_name='rlon').cube_dims(cube) + ) + cube.add_aux_coord( + new_coord, + aux_coord_dims) def fix_metadata(self, cubes): """Fix CORDEX rotated grids. diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index 6416533067..b02ec58365 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -11,6 +11,7 @@ MOHCHadREM3GA705, TimeLongName, ) +from esmvalcore.exceptions import RecipeError @pytest.fixture @@ -148,14 +149,38 @@ def test_clmcomcclm4817_fix_metadata(cubes): def test_rotated_grid_fix(cordex_cubes): fix = AllVars( vardef=None, - extra_facets={'domain': 'EUR-11'}) - - out_cubes = fix.fix_metadata(cordex_cubes) + extra_facets={ + 'domain': 'EUR-11', + 'dataset': 'DATASET', + 'driver': 'DRIVER' + } + ) domain = cx.cordex_domain('EUR-11', add_vertices=True) - assert cordex_cubes is out_cubes - for cube in out_cubes: + for cube in cordex_cubes: for coord in ['rlat', 'rlon', 'lat', 'lon']: cube_coord = cube.coord(var_name=coord) + cube_coord.points = domain[coord].data + 1e-6 + out_cubes = fix.fix_metadata(cordex_cubes) + assert cordex_cubes is out_cubes + for out_cube in out_cubes: + for coord in ['rlat', 'rlon', 'lat', 'lon']: + cube_coord = out_cube.coord(var_name=coord) domain_coord = domain[coord].data np.testing.assert_array_equal( cube_coord.points, domain_coord) + +def test_rotated_grid_fix_error(cordex_cubes): + fix = AllVars( + vardef=None, + extra_facets={ + 'domain': 'EUR-11', + 'dataset': 'DATASET', + 'driver': 'DRIVER' + } + ) + msg = ("Differences between the original grid and the " + "standarised grid are above 10e-4 degrees.") + with pytest.raises(RecipeError) as exc: + fix.fix_metadata(cordex_cubes) + assert msg == exc.value.message + From 7cc3c70cd431b9aaabcc9dbab04000d9dc348484 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 14 Dec 2022 10:50:45 +0100 Subject: [PATCH 57/75] Fix flake --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 1 + tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 0abc3eeab7..0af86d6c04 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -10,6 +10,7 @@ from esmvalcore.exceptions import RecipeError logger = logging.getLogger(__name__) + class MOHCHadREM3GA705(Fix): """General fix for MOHC-HadREM3-GA7-05.""" diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index b02ec58365..f95627d8c1 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -169,6 +169,7 @@ def test_rotated_grid_fix(cordex_cubes): np.testing.assert_array_equal( cube_coord.points, domain_coord) + def test_rotated_grid_fix_error(cordex_cubes): fix = AllVars( vardef=None, @@ -183,4 +184,3 @@ def test_rotated_grid_fix_error(cordex_cubes): with pytest.raises(RecipeError) as exc: fix.fix_metadata(cordex_cubes) assert msg == exc.value.message - From e39bb4304393989f8bf4226fcd9d55519930b9c9 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 14 Dec 2022 13:12:48 +0100 Subject: [PATCH 58/75] Fix codacy --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 0af86d6c04..f84616fe92 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -1,11 +1,12 @@ """Fixes that are shared between datasets and drivers.""" +import logging from cf_units import Unit import cordex as cx import numpy as np import iris -import logging from iris.coord_systems import RotatedGeogCS +from esmvalcore.cmor.check import CMORCheck from esmvalcore.cmor.fix import Fix from esmvalcore.exceptions import RecipeError logger = logging.getLogger(__name__) @@ -117,9 +118,9 @@ def _check_grid_differences(self, old_coord, new_coord): if diff > 10e-4: raise RecipeError( - "Differences between the original grid and the " - f"standarised grid are above 10e-4 {new_coord.units}.", - ) + "Differences between the original grid and the " + f"standarised grid are above 10e-4 {new_coord.units}.", + ) def _fix_rotated_coords(self, cube): """Fix rotated coordinates.""" @@ -164,7 +165,6 @@ def _fix_geographical_coords(self, cube): units=Unit(domain[aux_coord].units), bounds=bounds ) - logger.info(old_coord.units) self._check_grid_differences(old_coord, new_coord) aux_coord_dims = ( cube.coord(var_name='rlat').cube_dims(cube) + From 33f9e295bbb134654f3df51f14c0a6c6480185e4 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 14 Dec 2022 14:26:47 +0100 Subject: [PATCH 59/75] Remove unused import --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index f84616fe92..7ab04649ea 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -6,7 +6,6 @@ import iris from iris.coord_systems import RotatedGeogCS -from esmvalcore.cmor.check import CMORCheck from esmvalcore.cmor.fix import Fix from esmvalcore.exceptions import RecipeError logger = logging.getLogger(__name__) From 6e52386186986108096cc2bb3fc0f1fafd3ceed8 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 14 Dec 2022 16:13:15 +0100 Subject: [PATCH 60/75] Format log message properly --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 7ab04649ea..581c7feb1a 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -99,21 +99,14 @@ def fix_metadata(self, cubes): class AllVars(Fix): """General CORDEX grid fix.""" - _grid_diff_msg = ('Maximum difference between original {} ' - 'points and standard {} domain points ' - 'for dataset {} and driver {} is: {}') - def _check_grid_differences(self, old_coord, new_coord): """Check differences between coords.""" diff = np.max(np.abs(old_coord.points - new_coord.points)) logger.debug( - self._grid_diff_msg, - new_coord.var_name, - self.extra_facets['domain'], - self.extra_facets['dataset'], - self.extra_facets['driver'], - diff - ) + f"Maximum difference between original {new_coord.var_name} " + f"points and standard {self.extra_facets['domain']} domain points " + f"for dataset {self.extra_facets['dataset']} and " + f"driver {self.extra_facets['driver']} is: {diff}.") if diff > 10e-4: raise RecipeError( From 40692afb635a01b869f3826fe08038f6819cd649 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Wed, 14 Dec 2022 16:13:46 +0100 Subject: [PATCH 61/75] Add dataset in extra_facets dict for cordex --- esmvalcore/cmor/_fixes/fix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index d3766007d8..f7239a7a33 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -161,6 +161,7 @@ def get_fixes(project, dataset, mip, short_name, extra_facets=None): fixes_modules = [] if project == 'cordex': driver = extra_facets['driver'].replace('-', '_').lower() + extra_facets['dataset'] = dataset try: fixes_modules.append(importlib.import_module( f'esmvalcore.cmor._fixes.{project}.{driver}.{dataset}' From e54e86209017b24492aed2778a432dd2f8f2bb38 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 16 Dec 2022 11:56:09 +0100 Subject: [PATCH 62/75] Add warning for Lambert Conformal datasets --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 22 ++++++++++++++----- .../cmor/_fixes/cordex/test_cordex_fixes.py | 18 +++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 581c7feb1a..2ae14a169e 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -5,7 +5,7 @@ import numpy as np import iris -from iris.coord_systems import RotatedGeogCS +from iris.coord_systems import RotatedGeogCS, LambertConformal from esmvalcore.cmor.fix import Fix from esmvalcore.exceptions import RecipeError logger = logging.getLogger(__name__) @@ -103,10 +103,15 @@ def _check_grid_differences(self, old_coord, new_coord): """Check differences between coords.""" diff = np.max(np.abs(old_coord.points - new_coord.points)) logger.debug( - f"Maximum difference between original {new_coord.var_name} " - f"points and standard {self.extra_facets['domain']} domain points " - f"for dataset {self.extra_facets['dataset']} and " - f"driver {self.extra_facets['driver']} is: {diff}.") + "Maximum difference between original %s" + "points and standard %s domain points " + "for dataset %s and driver %s is: %s.", + new_coord.var_name, + self.extra_facets['domain'], + self.extra_facets['dataset'], + self.extra_facets['driver'], + str(diff) + ) if diff > 10e-4: raise RecipeError( @@ -190,5 +195,12 @@ def fix_metadata(self, cubes): if isinstance(coord_system, RotatedGeogCS): self._fix_rotated_coords(cube) self._fix_geographical_coords(cube) + if isinstance(coord_system, LambertConformal): + logger.warning( + "Support for CORDEX datasets in a Lambert Conformal " + "coordinate system is ongoing. Certain preprocessor " + "functions may fail." + ) + return cubes diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index f95627d8c1..45d0325632 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -184,3 +184,21 @@ def test_rotated_grid_fix_error(cordex_cubes): with pytest.raises(RecipeError) as exc: fix.fix_metadata(cordex_cubes) assert msg == exc.value.message + + +def test_lambert_grid_warning(cubes, caplog): + fix = AllVars( + vardef=None, + extra_facets={ + 'domain': 'EUR-11', + 'dataset': 'DATASET', + 'driver': 'DRIVER' + } + ) + for cube in cubes: + cube.coord_system = iris.coord_systems.LambertConformal + fix.fix_metadata(cubes) + msg = ("Support for CORDEX datasets in a Lambert Conformal " + "coordinate system is ongoing. Certain preprocessor " + "functions may fail.") + assert msg in caplog.text From 67c877fedee508407bba527594f85e0d37d9cbe3 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Fri, 16 Dec 2022 12:04:50 +0100 Subject: [PATCH 63/75] Remove blank line --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 2ae14a169e..3865f93dc6 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -202,5 +202,4 @@ def fix_metadata(self, cubes): "functions may fail." ) - return cubes From af968d338c9ecf01afafcf989f6a3cceec3a9f3c Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Mon, 19 Dec 2022 09:47:28 +0100 Subject: [PATCH 64/75] Update esmvalcore/cmor/_fixes/cordex/cordex_fixes.py Co-authored-by: Klaus Zimmermann --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 3865f93dc6..696ad097dd 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -160,7 +160,7 @@ def _fix_geographical_coords(self, cube): standard_name=domain[aux_coord].standard_name, long_name=domain[aux_coord].long_name, units=Unit(domain[aux_coord].units), - bounds=bounds + bounds=bounds, ) self._check_grid_differences(old_coord, new_coord) aux_coord_dims = ( From ed89cc4a38ae6a8d8f525df313505a5411b93cd3 Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Mon, 19 Dec 2022 09:47:36 +0100 Subject: [PATCH 65/75] Update tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py Co-authored-by: Klaus Zimmermann --- tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index 45d0325632..514ff5db11 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -63,7 +63,7 @@ def cubes(): def cordex_cubes(): coord_system = iris.coord_systems.RotatedGeogCS( grid_north_pole_latitude=39.25, - grid_north_pole_longitude=-162 + grid_north_pole_longitude=-162, ) time = iris.coords.DimCoord(np.arange(0, 3), var_name='time', From 6112c96f70a0c240b69189c0df47d1185be8f7fd Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 19 Dec 2022 10:46:34 +0100 Subject: [PATCH 66/75] Address review comments --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 24 +++++++++++-------- esmvalcore/cmor/check.py | 2 ++ .../cmor/_fixes/cordex/test_cordex_fixes.py | 19 +++++++++++++++ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 3865f93dc6..91ee49578e 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -119,11 +119,8 @@ def _check_grid_differences(self, old_coord, new_coord): f"standarised grid are above 10e-4 {new_coord.units}.", ) - def _fix_rotated_coords(self, cube): + def _fix_rotated_coords(self, cube, domain, domain_info): """Fix rotated coordinates.""" - data_domain = self.extra_facets['domain'] - domain = cx.cordex_domain(data_domain, add_vertices=True) - domain_info = cx.domain_info(data_domain) for dim_coord in ['rlat', 'rlon']: old_coord = cube.coord(domain[dim_coord].standard_name) old_coord_dims = old_coord.cube_dims(cube) @@ -145,10 +142,8 @@ def _fix_rotated_coords(self, cube): cube.remove_coord(old_coord) cube.add_dim_coord(new_coord, old_coord_dims) - def _fix_geographical_coords(self, cube): + def _fix_geographical_coords(self, cube, domain): """Fix geographical coordinates.""" - data_domain = self.extra_facets['domain'] - domain = cx.cordex_domain(data_domain, add_vertices=True) for aux_coord in ['lat', 'lon']: old_coord = cube.coord(domain[aux_coord].standard_name) cube.remove_coord(old_coord) @@ -190,16 +185,25 @@ def fix_metadata(self, cubes): iris.cube.CubeList """ + data_domain = self.extra_facets['domain'] + domain = cx.cordex_domain(data_domain, add_vertices=True) + domain_info = cx.domain_info(data_domain) for cube in cubes: coord_system = cube.coord_system() if isinstance(coord_system, RotatedGeogCS): - self._fix_rotated_coords(cube) - self._fix_geographical_coords(cube) - if isinstance(coord_system, LambertConformal): + self._fix_rotated_coords(cube, domain, domain_info) + self._fix_geographical_coords(cube, domain) + elif isinstance(coord_system, LambertConformal): logger.warning( "Support for CORDEX datasets in a Lambert Conformal " "coordinate system is ongoing. Certain preprocessor " "functions may fail." ) + else: + raise RecipeError( + f"Coordinate system {coord_system.grid_mapping_name} " + "not supported in CORDEX datasets. Must be " + "rotated_latitude_longitude or lambert_conformal_conic.", + ) return cubes diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index aea988fbcd..328e9ce374 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -1082,6 +1082,8 @@ def _get_cmor_checker(table, cmor_table = CMOR_TABLES[table] if table == 'CORDEX' and mip.endswith('hr'): + # CORDEX X-hourly tables define the mip + # as ending in 'h' instead of 'hr'. mip = mip.replace('hr', 'h') var_info = cmor_table.get_variable(mip, short_name) if var_info is None: diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index 45d0325632..7a86dacf98 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -202,3 +202,22 @@ def test_lambert_grid_warning(cubes, caplog): "coordinate system is ongoing. Certain preprocessor " "functions may fail.") assert msg in caplog.text + + +def test_wrong_coord_system(cubes): + fix = AllVars( + vardef=None, + extra_facets={ + 'domain': 'EUR-11', + 'dataset': 'DATASET', + 'driver': 'DRIVER' + } + ) + for cube in cubes: + cube.coord_system = iris.coord_systems.AlbersEqualArea + msg = ("Coordinate system albers_conical_equal_area not supported in " + "CORDEX datasets. Must be rotated_latitude_longitude " + "or lambert_conformal_conic.") + with pytest.raises(RecipeError) as exc: + fix.fix_metadata(cubes) + assert msg == exc.value.message From 1b89aa21173f443d2bc2650712008ef62a86ba0a Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 19 Dec 2022 10:59:28 +0100 Subject: [PATCH 67/75] Add comment in init file --- tests/integration/cmor/_fixes/cordex/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/cmor/_fixes/cordex/__init__.py b/tests/integration/cmor/_fixes/cordex/__init__.py index e69de29bb2..ed5327330d 100644 --- a/tests/integration/cmor/_fixes/cordex/__init__.py +++ b/tests/integration/cmor/_fixes/cordex/__init__.py @@ -0,0 +1 @@ +"""Integration tests for CORDEX fixes.""" \ No newline at end of file From ab74aeb14c635a44d1a87bc41d4704ae86559626 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 19 Dec 2022 12:40:26 +0100 Subject: [PATCH 68/75] Fix flake --- tests/integration/cmor/_fixes/cordex/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cmor/_fixes/cordex/__init__.py b/tests/integration/cmor/_fixes/cordex/__init__.py index ed5327330d..7aed60134a 100644 --- a/tests/integration/cmor/_fixes/cordex/__init__.py +++ b/tests/integration/cmor/_fixes/cordex/__init__.py @@ -1 +1 @@ -"""Integration tests for CORDEX fixes.""" \ No newline at end of file +"""Integration tests for CORDEX fixes.""" From c7cda806af3d9dd0574ea73bb9303998186033d4 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 19 Dec 2022 16:29:36 +0100 Subject: [PATCH 69/75] Improve test --- .../cmor/_fixes/cordex/test_cordex_fixes.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index f1884757cd..0946268543 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -101,15 +101,20 @@ def cordex_cubes(): ) return iris.cube.CubeList([cube]) - -def test_mohchadrem3ga705_fix_metadata(cubes): +@pytest.mark.parametrize( + 'coord, var_name, long_name', + [ + ('time', 'time', 'time'), + ('latitude','lat', 'latitude'), + ('longitude', 'lon', 'longitude'), + ]) +def test_mohchadrem3ga705_fix_metadata(cubes, coord, var_name, long_name): fix = MOHCHadREM3GA705(None) out_cubes = fix.fix_metadata(cubes) assert cubes is out_cubes for cube in out_cubes: - for coord in cube.coords(): - right_coord = cubes[0].coord(coord) - assert coord == right_coord + assert cube.coord(standard_name=coord).var_name == var_name + assert cube.coord(standard_name=coord).long_name == long_name def test_timelongname_fix_metadata(cubes): From 33c5d1e18c0b665b22e61ffe7ffb35b53b174f32 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 19 Dec 2022 16:34:36 +0100 Subject: [PATCH 70/75] Safer calendar change --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index a494bf5e40..d16a1c91f1 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -81,11 +81,9 @@ def fix_metadata(self, cubes): """ for cube in cubes: - if cube.coord('time').units.calendar == 'standard': - cube.coord('time').units = Unit( - 'days since 1850-1-1 00:00:00', - calendar='proleptic_gregorian' - ) + time_unit = cube.coord('time').units + if time_unit.calendar == 'standard': + time_unit.change_calendar('proleptic_gregorian') for coord in cube.coords(): if coord.dtype in ['>f8', '>f4']: coord.points = coord.core_points().astype( From 3480152425c5559f8427cb88d3c09697c52dd2af Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 19 Dec 2022 16:41:00 +0100 Subject: [PATCH 71/75] Fix flake --- tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index 0946268543..bc1ee93f99 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -101,11 +101,12 @@ def cordex_cubes(): ) return iris.cube.CubeList([cube]) + @pytest.mark.parametrize( 'coord, var_name, long_name', [ ('time', 'time', 'time'), - ('latitude','lat', 'latitude'), + ('latitude', 'lat', 'latitude'), ('longitude', 'lon', 'longitude'), ]) def test_mohchadrem3ga705_fix_metadata(cubes, coord, var_name, long_name): From fb9f634aedabb638cd3bdbbe232ec704d97c1398 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 19 Dec 2022 17:01:48 +0100 Subject: [PATCH 72/75] Fix test --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index d16a1c91f1..43101b0868 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -83,7 +83,8 @@ def fix_metadata(self, cubes): for cube in cubes: time_unit = cube.coord('time').units if time_unit.calendar == 'standard': - time_unit.change_calendar('proleptic_gregorian') + new_unit = time_unit.change_calendar('proleptic_gregorian') + cube.coord('time').units = new_unit for coord in cube.coords(): if coord.dtype in ['>f8', '>f4']: coord.points = coord.core_points().astype( From 99c4214522856a766c937fc9dd202faa49a408b1 Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 19 Dec 2022 17:42:16 +0100 Subject: [PATCH 73/75] Cache domain retrieval --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index 43101b0868..bcf7dace4c 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -1,16 +1,31 @@ """Fixes that are shared between datasets and drivers.""" import logging -from cf_units import Unit +from functools import lru_cache + import cordex as cx -import numpy as np import iris +import numpy as np +from cf_units import Unit +from iris.coord_systems import LambertConformal, RotatedGeogCS -from iris.coord_systems import RotatedGeogCS, LambertConformal from esmvalcore.cmor.fix import Fix from esmvalcore.exceptions import RecipeError + logger = logging.getLogger(__name__) +@lru_cache +def _get_domain(data_domain): + domain = cx.cordex_domain(data_domain, add_vertices=True) + return domain + + +@lru_cache +def _get_domain_info(data_domain): + domain_info = cx.domain_info(data_domain) + return domain_info + + class MOHCHadREM3GA705(Fix): """General fix for MOHC-HadREM3-GA7-05.""" @@ -25,7 +40,6 @@ def fix_metadata(self, cubes): Returns ------- iris.cube.CubeList - """ for cube in cubes: cube.coord('latitude').var_name = 'lat' @@ -39,8 +53,7 @@ class TimeLongName(Fix): """Fixes for time coordinate.""" def fix_metadata(self, cubes): - """ - Fix time long_name. + """Fix time long_name. Parameters ---------- @@ -50,7 +63,6 @@ def fix_metadata(self, cubes): Returns ------- iris.cube.CubeList - """ for cube in cubes: cube.coord('time').long_name = 'time' @@ -78,7 +90,6 @@ def fix_metadata(self, cubes): Returns ------- iris.cube.CubeList - """ for cube in cubes: time_unit = cube.coord('time').units @@ -103,20 +114,15 @@ def _check_grid_differences(self, old_coord, new_coord): diff = np.max(np.abs(old_coord.points - new_coord.points)) logger.debug( "Maximum difference between original %s" - "points and standard %s domain points " - "for dataset %s and driver %s is: %s.", - new_coord.var_name, - self.extra_facets['domain'], - self.extra_facets['dataset'], - self.extra_facets['driver'], - str(diff) - ) + "points and standard %s domain points " + "for dataset %s and driver %s is: %s.", new_coord.var_name, + self.extra_facets['domain'], self.extra_facets['dataset'], + self.extra_facets['driver'], str(diff)) if diff > 10e-4: raise RecipeError( "Differences between the original grid and the " - f"standarised grid are above 10e-4 {new_coord.units}.", - ) + f"standardised grid are above 10e-4 {new_coord.units}.", ) def _fix_rotated_coords(self, cube, domain, domain_info): """Fix rotated coordinates.""" @@ -126,8 +132,7 @@ def _fix_rotated_coords(self, cube, domain, domain_info): points = domain[dim_coord].data coord_system = iris.coord_systems.RotatedGeogCS( grid_north_pole_latitude=domain_info['pollat'], - grid_north_pole_longitude=domain_info['pollon'] - ) + grid_north_pole_longitude=domain_info['pollon']) new_coord = iris.coords.DimCoord( points, var_name=dim_coord, @@ -157,13 +162,9 @@ def _fix_geographical_coords(self, cube, domain): bounds=bounds, ) self._check_grid_differences(old_coord, new_coord) - aux_coord_dims = ( - cube.coord(var_name='rlat').cube_dims(cube) + - cube.coord(var_name='rlon').cube_dims(cube) - ) - cube.add_aux_coord( - new_coord, - aux_coord_dims) + aux_coord_dims = (cube.coord(var_name='rlat').cube_dims(cube) + + cube.coord(var_name='rlon').cube_dims(cube)) + cube.add_aux_coord(new_coord, aux_coord_dims) def fix_metadata(self, cubes): """Fix CORDEX rotated grids. @@ -182,11 +183,10 @@ def fix_metadata(self, cubes): Returns ------- iris.cube.CubeList - """ data_domain = self.extra_facets['domain'] - domain = cx.cordex_domain(data_domain, add_vertices=True) - domain_info = cx.domain_info(data_domain) + domain = _get_domain(data_domain) + domain_info = _get_domain_info(data_domain) for cube in cubes: coord_system = cube.coord_system() if isinstance(coord_system, RotatedGeogCS): @@ -196,13 +196,11 @@ def fix_metadata(self, cubes): logger.warning( "Support for CORDEX datasets in a Lambert Conformal " "coordinate system is ongoing. Certain preprocessor " - "functions may fail." - ) + "functions may fail.") else: raise RecipeError( f"Coordinate system {coord_system.grid_mapping_name} " "not supported in CORDEX datasets. Must be " - "rotated_latitude_longitude or lambert_conformal_conic.", - ) + "rotated_latitude_longitude or lambert_conformal_conic.", ) return cubes From e5d48ccbb4f29cfee906657cb8304e7b8f9e318c Mon Sep 17 00:00:00 2001 From: sloosvel Date: Mon, 19 Dec 2022 17:49:30 +0100 Subject: [PATCH 74/75] Fix spelling --- tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py index bc1ee93f99..a831d37b32 100644 --- a/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py +++ b/tests/integration/cmor/_fixes/cordex/test_cordex_fixes.py @@ -186,7 +186,7 @@ def test_rotated_grid_fix_error(cordex_cubes): } ) msg = ("Differences between the original grid and the " - "standarised grid are above 10e-4 degrees.") + "standardised grid are above 10e-4 degrees.") with pytest.raises(RecipeError) as exc: fix.fix_metadata(cordex_cubes) assert msg == exc.value.message From 684ebc2c7606c53ba296e2b68dbc9dc77bec3501 Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Mon, 19 Dec 2022 17:50:46 +0100 Subject: [PATCH 75/75] Apply suggestions from code review Co-authored-by: Bouwe Andela --- esmvalcore/cmor/_fixes/cordex/cordex_fixes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py index bcf7dace4c..9ccbfd4ed3 100644 --- a/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py +++ b/esmvalcore/cmor/_fixes/cordex/cordex_fixes.py @@ -122,7 +122,7 @@ def _check_grid_differences(self, old_coord, new_coord): if diff > 10e-4: raise RecipeError( "Differences between the original grid and the " - f"standardised grid are above 10e-4 {new_coord.units}.", ) + f"standardised grid are above 10e-4 {new_coord.units}.") def _fix_rotated_coords(self, cube, domain, domain_info): """Fix rotated coordinates.""" @@ -201,6 +201,6 @@ def fix_metadata(self, cubes): raise RecipeError( f"Coordinate system {coord_system.grid_mapping_name} " "not supported in CORDEX datasets. Must be " - "rotated_latitude_longitude or lambert_conformal_conic.", ) + "rotated_latitude_longitude or lambert_conformal_conic.") return cubes