Combine ERA5 and ERA5T data

Requesting cutout data spanning recent (ERA5T)
and data older than ~3 months (ERA5) results in an
additional dimension in `cutout.data`, called `expver`,
which `atlite` currently cannot handle gracefully.

This change collapses the two dimensions into a single dimension.

See discussion in PyPSA#190

atlite/data.py

@@ -108,7 +108,9 @@ def wrapper(*args, **kwargs):
 
 
 @maybe_remove_tmpdir
-def cutout_prepare(cutout, features=None, tmpdir=None, overwrite=False):
+def cutout_prepare(
+    cutout, features=None, tmpdir=None, overwrite=False, flatten_expver=True
+):
     """
     Prepare all or a selection of features in a cutout.
 
@@ -133,6 +135,11 @@ def cutout_prepare(cutout, features=None, tmpdir=None, overwrite=False):
     overwrite : bool, optional
         Whether to overwrite variables which are already included in the
         cutout. The default is False.
+    flatten_expver: bool, default True
+        If true, data for time periods spanning both ERA5 and ERA5T data
+        will be combined into a single dimension.
+        See https://confluence.ecmwf.int/pages/viewpage.action?pageId=173385064
+        for details.
 
     Returns
     -------

atlite/datasets/era5.py

@@ -91,6 +91,13 @@ def _rename_and_clean_coords(ds, add_lon_lat=True):
     ds = maybe_swap_spatial_dims(ds)
     if add_lon_lat:
         ds = ds.assign_coords(lon=ds.coords["x"], lat=ds.coords["y"])
+
+    # Combine ERA5 and ERA5T data into a single dimension.
+    # See https://github.com/PyPSA/atlite/issues/190
+    if "expver" in ds.dims.keys():
+        ds = ds.sel(expver=1).combine_first(ds.sel(expver=5))
+        # ds = ds.reduce(np.nansum, 'expver')
+
     return ds
 
 

test/test_preparation_and_conversion.py

@@ -37,7 +37,7 @@ def all_notnull_test(cutout):
 
 def prepared_features_test(cutout):
     """
-    The prepared features series should contain all variables in cuttout.data
+    The prepared features series should contain all variables in cutout.data
     """
     assert set(cutout.prepared_features) == set(cutout.data)
 
@@ -293,7 +293,7 @@ def coefficient_of_performance_test(cutout):
 # %% Prepare cutouts to test
 
 
-TIME = "2013-01-01"
+TIME = "2022-07-31"
 BOUNDS = (-4, 56, 1.5, 62)
 SARAH_DIR = os.getenv("SARAH_DIR", "/home/vres/climate-data/sarah_v2")
 GEBCO_PATH = os.getenv("GEBCO_PATH", "/home/vres/climate-data/GEBCO_2014_2D.nc")
@@ -307,6 +307,16 @@ def cutout_era5(tmp_path_factory):
     return cutout
 
 
+@pytest.fixture(scope="session")
+def cutout_mixed_era5_and_era5t(tmp_path_factory):
+    # Try just getting Berlin
+    # Try just requesting era5 and era5t
+    tmp_path = tmp_path_factory.mktemp("era5_era5t")
+    cutout = Cutout(path=tmp_path / "era5_era5t", module="era5", bounds=BOUNDS, time=slice("2022-07-31", "2022-08-01"))
+    cutout.prepare()
+    return cutout
+
+
 @pytest.fixture(scope="session")
 def cutout_era5_coarse(tmp_path_factory):
     tmp_path = tmp_path_factory.mktemp("era5_coarse")
@@ -403,7 +413,7 @@ class TestERA5:
     @staticmethod
     def test_data_module_arguments_era5(cutout_era5):
         """
-        All data variables should have an attribute to which module thay belong
+        All data variables should have an attribute to which module they belong
         """
         for v in cutout_era5.data:
             assert cutout_era5.data.attrs["module"] == "era5"
@@ -490,6 +500,10 @@ def test_wrong_loading(cutout_era5):
     def test_pv_era5(cutout_era5):
         return pv_test(cutout_era5)
 
+    @staticmethod
+    def test_pv_era5(cutout_mixed_era5_and_era5t):
+        return pv_test(cutout_mixed_era5_and_era5t)
+
     @staticmethod
     def test_wind_era5(cutout_era5):
         return wind_test(cutout_era5)