From f27a91d59ff86633ac5578e9e6b9f27fcff80574 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Fri, 17 May 2024 07:02:34 +0000 Subject: [PATCH 01/29] fix for file reading; includes removing chunk reading and decoding times after import --- satpy/readers/mviri_l1b_fiduceo_nc.py | 26 ++++++++++++++----- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 20 ++++++++++---- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index fc5aea2c8e..0bbc285917 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -153,10 +153,12 @@ """ import abc +import datetime import functools import warnings import dask.array as da +import numpy import numpy as np import xarray as xr @@ -513,11 +515,20 @@ def get_time(self): """Get time coordinate. Variable is sometimes named "time" and sometimes "time_ir_wv". + FillValues in time are set to NaT. """ + try: - return self["time_ir_wv"] + time = self["time_ir_wv"] except KeyError: - return self["time"] + time = self["time"] + + timeOffset = time.attrs["add_offset"] + condition = time == time.attrs["_FillValue"] + + time = xr.where(condition, numpy.nan, time + timeOffset) + time = (time*1e9).astype("datetime64[ns]") + return time def get_xy_coords(self, resolution): """Get x and y coordinates for the given resolution.""" @@ -555,11 +566,14 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 self.mask_bad_quality = mask_bad_quality nc_raw = xr.open_dataset( filename, - chunks={"x": CHUNK_SIZE, - "y": CHUNK_SIZE, - "x_ir_wv": CHUNK_SIZE, - "y_ir_wv": CHUNK_SIZE} + # chunks={"x": CHUNK_SIZE, + # "y": CHUNK_SIZE, + # "x_ir_wv": CHUNK_SIZE, + # "y_ir_wv": CHUNK_SIZE}, + decode_times=False, + decode_cf=False ) + self.nc = DatasetWrapper(nc_raw) # Projection longitude is not provided in the file, read it from the diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 04694c145a..3c50ebcd43 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -19,6 +19,7 @@ from __future__ import annotations +import datetime import os from unittest import mock @@ -61,8 +62,8 @@ {"sun_earth_distance_correction_applied": True, "sun_earth_distance_correction_factor": 1.} ) -acq_time_vis_exp = [np.datetime64("1970-01-01 00:30").astype("datetime64[ns]"), - np.datetime64("1970-01-01 00:30").astype("datetime64[ns]"), +acq_time_vis_exp = [np.datetime64("NaT").astype("datetime64[ns]"), + np.datetime64("NaT").astype("datetime64[ns]"), np.datetime64("1970-01-01 02:30").astype("datetime64[ns]"), np.datetime64("1970-01-01 02:30").astype("datetime64[ns]")] vis_counts_exp = xr.DataArray( @@ -124,7 +125,7 @@ }, attrs=attrs_exp ) -acq_time_ir_wv_exp = [np.datetime64("1970-01-01 00:30").astype("datetime64[ns]"), +acq_time_ir_wv_exp = [np.datetime64("NaT").astype("datetime64[ns]"), np.datetime64("1970-01-01 02:30").astype("datetime64[ns]")] wv_counts_exp = xr.DataArray( np.array( @@ -272,8 +273,13 @@ def fixture_fake_dataset(): dtype=np.uint8 ) ) - time = np.arange(4) * 60 * 60 * 1e9 - time = time.astype("datetime64[ns]").reshape(2, 2) + time = np.arange(4) * 60 * 60 + timeFillValue=4294967295 + timeAddOffset=0 + time[0] = timeFillValue + time[1] = timeFillValue + time = time.reshape(2,2) + ds = xr.Dataset( data_vars={ "count_vis": (("y", "x"), count_vis), @@ -317,6 +323,10 @@ def fixture_fake_dataset(): ) ds["count_ir"].attrs["ancillary_variables"] = "a_ir b_ir" ds["count_wv"].attrs["ancillary_variables"] = "a_wv b_wv" + + ds["time_ir_wv"].attrs["_FillValue"] = timeFillValue + ds["time_ir_wv"].attrs["add_offset"] = timeAddOffset + return ds From 2dbae1af4d8d48638b7c0c7683c356aee071073e Mon Sep 17 00:00:00 2001 From: bkremmli Date: Fri, 17 May 2024 08:23:37 +0000 Subject: [PATCH 02/29] remove import datetime --- satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 3c50ebcd43..3cb6070fed 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -19,7 +19,6 @@ from __future__ import annotations -import datetime import os from unittest import mock From eb0382a77471505a1170bf6b3c1471edd59ebeb7 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Fri, 17 May 2024 09:04:13 +0000 Subject: [PATCH 03/29] add bkremmli to AUTHORS.md --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index fb43d0168d..ec40df4309 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -44,6 +44,7 @@ The following people have made contributions to this project: - [Johannes Johansson (JohannesSMHI)](https://github.com/JohannesSMHI) - [Sauli Joro (sjoro)](https://github.com/sjoro) - [Janne Kotro (jkotro)](https://github.com/jkotro) +- [Beke Kremmling (bkremmli)](https://github.com/bkremmli) - Deutscher Wetterdienst - [Ralph Kuehn (ralphk11)](https://github.com/ralphk11) - [Panu Lahtinen (pnuu)](https://github.com/pnuu) - [Jussi Leinonen (jleinonen)](https://github.com/jleinonen) - meteoswiss From 7d6608a34d1300e845f9e7936c085e638b5966f1 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Fri, 17 May 2024 09:11:59 +0000 Subject: [PATCH 04/29] correct for failures from hook id ruff --- satpy/readers/mviri_l1b_fiduceo_nc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index 0bbc285917..fd86273e62 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -153,7 +153,6 @@ """ import abc -import datetime import functools import warnings @@ -517,7 +516,6 @@ def get_time(self): Variable is sometimes named "time" and sometimes "time_ir_wv". FillValues in time are set to NaT. """ - try: time = self["time_ir_wv"] except KeyError: From ec22136ae33d6f05d7f87e55fcaee09312b29233 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Fri, 17 May 2024 10:18:25 +0000 Subject: [PATCH 05/29] minor adaptations from PR comments --- satpy/readers/mviri_l1b_fiduceo_nc.py | 5 ++--- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 14 +++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index fd86273e62..b17f75fef0 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -157,7 +157,6 @@ import warnings import dask.array as da -import numpy import numpy as np import xarray as xr @@ -521,10 +520,10 @@ def get_time(self): except KeyError: time = self["time"] - timeOffset = time.attrs["add_offset"] + time_offset = time.attrs["add_offset"] condition = time == time.attrs["_FillValue"] - time = xr.where(condition, numpy.nan, time + timeOffset) + time = xr.where(condition, np.nan, time + time_offset) time = (time*1e9).astype("datetime64[ns]") return time diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 3cb6070fed..cf9573737c 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -273,11 +273,11 @@ def fixture_fake_dataset(): ) ) time = np.arange(4) * 60 * 60 - timeFillValue=4294967295 - timeAddOffset=0 - time[0] = timeFillValue - time[1] = timeFillValue - time = time.reshape(2,2) + time_fill_value = 4294967295 + time_add_offset = 0 + time[0] = time_fill_value + time[1] = time_fill_value + time = time.reshape(2, 2) ds = xr.Dataset( data_vars={ @@ -323,8 +323,8 @@ def fixture_fake_dataset(): ds["count_ir"].attrs["ancillary_variables"] = "a_ir b_ir" ds["count_wv"].attrs["ancillary_variables"] = "a_wv b_wv" - ds["time_ir_wv"].attrs["_FillValue"] = timeFillValue - ds["time_ir_wv"].attrs["add_offset"] = timeAddOffset + ds["time_ir_wv"].attrs["_FillValue"] = time_fill_value + ds["time_ir_wv"].attrs["add_offset"] = time_add_offset return ds From 5beedea8e90c30f133eccf7ec12c972f2abae3bf Mon Sep 17 00:00:00 2001 From: bkremmli <157395108+bkremmli@users.noreply.github.com> Date: Fri, 17 May 2024 12:20:19 +0200 Subject: [PATCH 06/29] Update satpy/readers/mviri_l1b_fiduceo_nc.py Co-authored-by: Martin Raspaud --- satpy/readers/mviri_l1b_fiduceo_nc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index b17f75fef0..b2ad3fd22a 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -524,7 +524,7 @@ def get_time(self): condition = time == time.attrs["_FillValue"] time = xr.where(condition, np.nan, time + time_offset) - time = (time*1e9).astype("datetime64[ns]") + time = time.astype("datetime64[s]").astype("datetime64[ns]") return time def get_xy_coords(self, resolution): From 21679c61f9f9c26e8ce4b6f756e735f9fc52fd7b Mon Sep 17 00:00:00 2001 From: bkremmli Date: Fri, 24 May 2024 08:02:54 +0000 Subject: [PATCH 07/29] perform chunking after open_dataset and use decode_cf = False --- satpy/readers/mviri_l1b_fiduceo_nc.py | 30 +++++++++++-------- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 16 +++++----- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index b17f75fef0..76c3a924c4 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -162,9 +162,7 @@ from satpy.readers._geos_area import get_area_definition, get_area_extent, sampling_to_lfac_cfac from satpy.readers.file_handlers import BaseFileHandler -from satpy.utils import get_legacy_chunk_size -CHUNK_SIZE = get_legacy_chunk_size() EQUATOR_RADIUS = 6378140.0 POLE_RADIUS = 6356755.0 ALTITUDE = 42164000.0 - EQUATOR_RADIUS @@ -458,6 +456,22 @@ class DatasetWrapper: def __init__(self, nc): """Wrap the given dataset.""" self.nc = nc + self._fix_duplicate_dimensions(nc) + self.nc = self._chunk(nc) + + def _fix_duplicate_dimensions(self, nc): + nc.variables["covariance_spectral_response_function_vis"].dims = ("srf_size_1", "srf_size_2") + + def _chunk(self, nc): + + (CHUNK_SIZE, CHUNK_SIZE) = nc.variables["quality_pixel_bitmask"].encoding["chunksizes"] + chunks = { + "x": CHUNK_SIZE, + "y": CHUNK_SIZE, + "x_ir_wv": CHUNK_SIZE, + "y_ir_wv": CHUNK_SIZE + } + return nc.chunk(chunks) @property def attrs(self): @@ -520,10 +534,6 @@ def get_time(self): except KeyError: time = self["time"] - time_offset = time.attrs["add_offset"] - condition = time == time.attrs["_FillValue"] - - time = xr.where(condition, np.nan, time + time_offset) time = (time*1e9).astype("datetime64[ns]") return time @@ -563,12 +573,8 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 self.mask_bad_quality = mask_bad_quality nc_raw = xr.open_dataset( filename, - # chunks={"x": CHUNK_SIZE, - # "y": CHUNK_SIZE, - # "x_ir_wv": CHUNK_SIZE, - # "y_ir_wv": CHUNK_SIZE}, - decode_times=False, - decode_cf=False + decode_cf=True, + decode_times=False ) self.nc = DatasetWrapper(nc_raw) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index cf9573737c..7a54a8d7ab 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -79,6 +79,7 @@ }, attrs=attrs_exp ) + vis_rad_exp = xr.DataArray( np.array( [[np.nan, 18.56, 38.28, 58.], @@ -272,11 +273,11 @@ def fixture_fake_dataset(): dtype=np.uint8 ) ) - time = np.arange(4) * 60 * 60 - time_fill_value = 4294967295 - time_add_offset = 0 - time[0] = time_fill_value - time[1] = time_fill_value + + cov = da.from_array([[1, 2], [3, 4]]) + time = np.arange(4) * 60 * 60. + time[0] = np.nan + time[1] = np.nan time = time.reshape(2, 2) ds = xr.Dataset( @@ -308,6 +309,7 @@ def fixture_fake_dataset(): "sub_satellite_longitude_end": np.nan, "sub_satellite_latitude_start": np.nan, "sub_satellite_latitude_end": 0.1, + "covariance_spectral_response_function_vis": (("srf_size", "srf_size"), cov), }, coords={ "y": [1, 2, 3, 4], @@ -322,9 +324,7 @@ def fixture_fake_dataset(): ) ds["count_ir"].attrs["ancillary_variables"] = "a_ir b_ir" ds["count_wv"].attrs["ancillary_variables"] = "a_wv b_wv" - - ds["time_ir_wv"].attrs["_FillValue"] = time_fill_value - ds["time_ir_wv"].attrs["add_offset"] = time_add_offset + ds["quality_pixel_bitmask"].encoding["chunksizes"] = (2, 2) return ds From 59880ced84accdfe8a3996419d57ff2befa9b060 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 28 May 2024 12:15:40 +0000 Subject: [PATCH 08/29] decode times separatly from other variables, adds TestInterpolator --- satpy/readers/mviri_l1b_fiduceo_nc.py | 32 +++++--- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 73 ++++++++++++++++++- 2 files changed, 90 insertions(+), 15 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index 76c3a924c4..fcf6c67c77 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -456,20 +456,27 @@ class DatasetWrapper: def __init__(self, nc): """Wrap the given dataset.""" self.nc = nc - self._fix_duplicate_dimensions(nc) - self.nc = self._chunk(nc) + + # remove time before decoding and add again. + raw_time = nc["time_ir_wv"] + self.nc = self.nc.drop_vars(["time_ir_wv"]) + self.nc = xr.decode_cf(self.nc) + self.nc["time_ir_wv"] = raw_time + + self._fix_duplicate_dimensions(self.nc) + self.nc = self._chunk(self.nc) def _fix_duplicate_dimensions(self, nc): nc.variables["covariance_spectral_response_function_vis"].dims = ("srf_size_1", "srf_size_2") def _chunk(self, nc): - (CHUNK_SIZE, CHUNK_SIZE) = nc.variables["quality_pixel_bitmask"].encoding["chunksizes"] + (chunk_size, chunk_size) = nc.variables["quality_pixel_bitmask"].encoding["chunksizes"] chunks = { - "x": CHUNK_SIZE, - "y": CHUNK_SIZE, - "x_ir_wv": CHUNK_SIZE, - "y_ir_wv": CHUNK_SIZE + "x": chunk_size, + "y": chunk_size, + "x_ir_wv": chunk_size, + "y_ir_wv": chunk_size } return nc.chunk(chunks) @@ -527,14 +534,16 @@ def get_time(self): """Get time coordinate. Variable is sometimes named "time" and sometimes "time_ir_wv". - FillValues in time are set to NaT. + FillValues in time are set to NaT, others converted to datetime64. """ try: time = self["time_ir_wv"] except KeyError: time = self["time"] - time = (time*1e9).astype("datetime64[ns]") + time = xr.where(time == time.attrs["_FillValue"], np.datetime64("NaT"), + (time + time.attrs["add_offset"]).astype("datetime64[s]").astype("datetime64[ns]")) + return time def get_xy_coords(self, resolution): @@ -573,8 +582,9 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 self.mask_bad_quality = mask_bad_quality nc_raw = xr.open_dataset( filename, - decode_cf=True, - decode_times=False + decode_cf=False, + decode_times=False, + mask_and_scale=False ) self.nc = DatasetWrapper(nc_raw) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 7a54a8d7ab..470e2d4b6f 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -28,6 +28,7 @@ import xarray as xr from pyproj import CRS from pyresample.geometry import AreaDefinition +from pytest_lazyfixture import lazy_fixture from satpy.readers.mviri_l1b_fiduceo_nc import ( ALTITUDE, @@ -36,6 +37,7 @@ DatasetWrapper, FiduceoMviriEasyFcdrFileHandler, FiduceoMviriFullFcdrFileHandler, + Interpolator, ) from satpy.tests.utils import make_dataid @@ -125,7 +127,7 @@ }, attrs=attrs_exp ) -acq_time_ir_wv_exp = [np.datetime64("NaT").astype("datetime64[ns]"), +acq_time_ir_wv_exp = [np.datetime64("NaT"), np.datetime64("1970-01-01 02:30").astype("datetime64[ns]")] wv_counts_exp = xr.DataArray( np.array( @@ -275,9 +277,9 @@ def fixture_fake_dataset(): ) cov = da.from_array([[1, 2], [3, 4]]) - time = np.arange(4) * 60 * 60. - time[0] = np.nan - time[1] = np.nan + time = np.arange(4) * 60 * 60 + time[0] = 4294967295 + time[1] = 4294967295 time = time.reshape(2, 2) ds = xr.Dataset( @@ -325,6 +327,8 @@ def fixture_fake_dataset(): ds["count_ir"].attrs["ancillary_variables"] = "a_ir b_ir" ds["count_wv"].attrs["ancillary_variables"] = "a_wv b_wv" ds["quality_pixel_bitmask"].encoding["chunksizes"] = (2, 2) + ds["time_ir_wv"].attrs["_FillValue"] = 4294967295 + ds["time_ir_wv"].attrs["add_offset"] = 0 return ds @@ -412,6 +416,7 @@ def test_get_dataset(self, file_handler, name, calibration, resolution, xr.testing.assert_allclose(ds, expected) assert ds.dtype == expected.dtype assert ds.attrs == expected.attrs + assert True def test_get_dataset_corrupt(self, file_handler): """Test getting datasets with known corruptions.""" @@ -600,3 +605,63 @@ def test_reassign_coords(self): ds = DatasetWrapper(nc) foo = ds["foo"] xr.testing.assert_equal(foo, foo_exp) + +class TestInterpolator: + """Unit tests for Interpolator class.""" + @pytest.fixture(name="time_ir_wv") + def fixture_time_ir_wv(self): + """Returns time_ir_wv.""" + return xr.DataArray( + [ + [np.datetime64("1970-01-01 01:00"), np.datetime64("1970-01-01 02:00")], + [np.datetime64("1970-01-01 03:00"), np.datetime64("1970-01-01 04:00")], + [np.datetime64("NaT"), np.datetime64("1970-01-01 06:00")], + [np.datetime64("NaT"), np.datetime64("NaT")], + ], + dims=("y", "x"), + coords={"y": [1, 3, 5, 7]} + ) + + @pytest.fixture(name="acq_time_vis_exp") + def fixture_acq_time_vis_exp(self): + """Returns acq_time_vis_exp.""" + return xr.DataArray( + [ + np.datetime64("1970-01-01 01:30"), + np.datetime64("1970-01-01 01:30"), + np.datetime64("1970-01-01 03:30"), + np.datetime64("1970-01-01 03:30"), + np.datetime64("1970-01-01 06:00"), + np.datetime64("1970-01-01 06:00"), + np.datetime64("NaT"), + np.datetime64("NaT") + ], + dims="y", + coords={"y": [1, 2, 3, 4, 5, 6, 7, 8]} + ) + + @pytest.fixture(name="acq_time_ir_exp") + def fixture_acq_time_ir_exp(self): + """Returns acq_time_ir_exp.""" + return xr.DataArray( + [ + np.datetime64("1970-01-01 01:30"), + np.datetime64("1970-01-01 03:30"), + np.datetime64("1970-01-01 06:00"), + np.datetime64("NaT"), + ], + dims="y", + coords={"y": [1, 3, 5, 7]} + ) + + @pytest.mark.parametrize( + "acq_time_exp", + [ + lazy_fixture("acq_time_ir_exp"), + lazy_fixture("acq_time_vis_exp") + ] + ) + def test_interp_acq_time(self, time_ir_wv, acq_time_exp): + """Tests time interpolation.""" + res = Interpolator.interp_acq_time(time_ir_wv, target_y=acq_time_exp.coords["y"]) + xr.testing.assert_allclose(res, acq_time_exp) From fb93f00b78adefed918ce122d78e43ff837b99a4 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 4 Jun 2024 13:23:57 +0000 Subject: [PATCH 09/29] fixes _decode_cf() and tests --- satpy/readers/mviri_l1b_fiduceo_nc.py | 41 +++++++++---------- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 6 +-- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index fcf6c67c77..4ef225ecc2 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -405,7 +405,6 @@ def interp_acq_time(time2d, target_y): """ # Compute mean timestamp per scanline time = time2d.mean(dim="x") - # If required, repeat timestamps in y-direction to obtain higher # resolution y = time.coords["y"].values @@ -453,30 +452,36 @@ def is_high_resol(resolution): class DatasetWrapper: """Helper class for accessing the dataset.""" - def __init__(self, nc): + def __init__(self, nc, decode_nc=True): """Wrap the given dataset.""" self.nc = nc + if decode_nc is True: + self._decode_cf() + self._fix_duplicate_dimensions(self.nc) + self.nc = self._chunk(self.nc) + + def _decode_cf(self): # remove time before decoding and add again. - raw_time = nc["time_ir_wv"] - self.nc = self.nc.drop_vars(["time_ir_wv"]) + time = self.get_time() + time_dims = self.nc["time_ir_wv"].dims + time = xr.where(time == time.attrs["_FillValue"], np.datetime64("NaT"), + (time + time.attrs["add_offset"]).astype("datetime64[s]").astype("datetime64[ns]")) + self.nc = self.nc.drop_vars(time.name) self.nc = xr.decode_cf(self.nc) - self.nc["time_ir_wv"] = raw_time - - self._fix_duplicate_dimensions(self.nc) - self.nc = self._chunk(self.nc) + self.nc[time.name] = (time_dims, time.values) def _fix_duplicate_dimensions(self, nc): nc.variables["covariance_spectral_response_function_vis"].dims = ("srf_size_1", "srf_size_2") def _chunk(self, nc): - (chunk_size, chunk_size) = nc.variables["quality_pixel_bitmask"].encoding["chunksizes"] + (chunk_size_y, chunk_size_x) = nc.variables["quality_pixel_bitmask"].encoding["chunksizes"] chunks = { - "x": chunk_size, - "y": chunk_size, - "x_ir_wv": chunk_size, - "y_ir_wv": chunk_size + "x": chunk_size_x, + "y": chunk_size_y, + "x_ir_wv": chunk_size_x, + "y_ir_wv": chunk_size_y } return nc.chunk(chunks) @@ -534,17 +539,11 @@ def get_time(self): """Get time coordinate. Variable is sometimes named "time" and sometimes "time_ir_wv". - FillValues in time are set to NaT, others converted to datetime64. """ try: - time = self["time_ir_wv"] + return self["time_ir_wv"] except KeyError: - time = self["time"] - - time = xr.where(time == time.attrs["_FillValue"], np.datetime64("NaT"), - (time + time.attrs["add_offset"]).astype("datetime64[s]").astype("datetime64[ns]")) - - return time + return self["time"] def get_xy_coords(self, resolution): """Get x and y coordinates for the given resolution.""" diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 470e2d4b6f..6a3d5525a8 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -28,7 +28,7 @@ import xarray as xr from pyproj import CRS from pyresample.geometry import AreaDefinition -from pytest_lazyfixture import lazy_fixture +from pytest_lazy_fixtures import lf as lazy_fixture from satpy.readers.mviri_l1b_fiduceo_nc import ( ALTITUDE, @@ -329,7 +329,6 @@ def fixture_fake_dataset(): ds["quality_pixel_bitmask"].encoding["chunksizes"] = (2, 2) ds["time_ir_wv"].attrs["_FillValue"] = 4294967295 ds["time_ir_wv"].attrs["add_offset"] = 0 - return ds @@ -416,7 +415,6 @@ def test_get_dataset(self, file_handler, name, calibration, resolution, xr.testing.assert_allclose(ds, expected) assert ds.dtype == expected.dtype assert ds.attrs == expected.attrs - assert True def test_get_dataset_corrupt(self, file_handler): """Test getting datasets with known corruptions.""" @@ -602,7 +600,7 @@ def test_reassign_coords(self): "x": [.3, .4] } ) - ds = DatasetWrapper(nc) + ds = DatasetWrapper(nc, decode_nc=False) foo = ds["foo"] xr.testing.assert_equal(foo, foo_exp) From 951c9b0cb4dcdc143250c4d1a7ab8ead83574252 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Wed, 5 Jun 2024 13:37:26 +0000 Subject: [PATCH 10/29] adds test_fix_duplicate_dimensions and removes leftover dimensions "srf_size" --- satpy/readers/mviri_l1b_fiduceo_nc.py | 3 ++- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index 4ef225ecc2..c170f87bd0 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -473,6 +473,7 @@ def _decode_cf(self): def _fix_duplicate_dimensions(self, nc): nc.variables["covariance_spectral_response_function_vis"].dims = ("srf_size_1", "srf_size_2") + self.nc = nc.drop_dims("srf_size") def _chunk(self, nc): @@ -583,7 +584,7 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 filename, decode_cf=False, decode_times=False, - mask_and_scale=False + mask_and_scale=False, ) self.nc = DatasetWrapper(nc_raw) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 6a3d5525a8..2cc7fdce61 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -329,6 +329,7 @@ def fixture_fake_dataset(): ds["quality_pixel_bitmask"].encoding["chunksizes"] = (2, 2) ds["time_ir_wv"].attrs["_FillValue"] = 4294967295 ds["time_ir_wv"].attrs["add_offset"] = 0 + return ds @@ -570,6 +571,30 @@ def test_file_pattern(self, reader): class TestDatasetWrapper: """Unit tests for DatasetWrapper class.""" + def test_fix_duplicate_dimensions(self): + """Test the renaming of duplicate dimensions. + + If duplicate dimensions are within the Dataset, opening the datasets with chunks throws an error. + Thus, the chunking needs to be done after opening the dataset and after the dimensions are renamed. + """ + foo = xr.Dataset( + data_vars={"covariance_spectral_response_function_vis": + (("srf_size", "srf_size"), [[1, 2], [3, 4]])} + ) + foo_ds = DatasetWrapper(foo, decode_nc=False) + foo_ds._fix_duplicate_dimensions(foo_ds.nc) + + foo_exp = xr.Dataset( + data_vars={"covariance_spectral_response_function_vis": + (("srf_size_1", "srf_size_2"), [[1, 2], [3, 4]])} + ) + + try: + foo_ds.nc.chunk("auto") + xr.testing.assert_allclose(foo_ds.nc, foo_exp) + except ValueError: + pytest.fail("Chunking failed.") + def test_reassign_coords(self): """Test reassigning of coordinates. From a379a086fbddf1de1c091591f3e621f7d1849ac3 Mon Sep 17 00:00:00 2001 From: bkremmli <157395108+bkremmli@users.noreply.github.com> Date: Wed, 5 Jun 2024 16:27:02 +0200 Subject: [PATCH 11/29] Update mviri_l1b_fiduceo_nc.py small fix to DataWrapper._decode_cf() --- satpy/readers/mviri_l1b_fiduceo_nc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index c170f87bd0..e9cd4eae28 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -464,7 +464,7 @@ def __init__(self, nc, decode_nc=True): def _decode_cf(self): # remove time before decoding and add again. time = self.get_time() - time_dims = self.nc["time_ir_wv"].dims + time_dims = self.nc[time.name].dims time = xr.where(time == time.attrs["_FillValue"], np.datetime64("NaT"), (time + time.attrs["add_offset"]).astype("datetime64[s]").astype("datetime64[ns]")) self.nc = self.nc.drop_vars(time.name) From 5ec0cf9eaa357f34f4362c06e3a86de26f46e5e3 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Thu, 6 Jun 2024 10:31:01 +0000 Subject: [PATCH 12/29] adds support for filenames of MVIRI FCDR L1.5 release 2 --- satpy/etc/readers/mviri_l1b_fiduceo_nc.yaml | 8 ++++++-- satpy/readers/mviri_l1b_fiduceo_nc.py | 9 +++++++-- satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 5 ++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/satpy/etc/readers/mviri_l1b_fiduceo_nc.yaml b/satpy/etc/readers/mviri_l1b_fiduceo_nc.yaml index ec3c5cab77..da30cb2545 100644 --- a/satpy/etc/readers/mviri_l1b_fiduceo_nc.yaml +++ b/satpy/etc/readers/mviri_l1b_fiduceo_nc.yaml @@ -20,14 +20,18 @@ file_types: nc_easy: file_reader: !!python/name:satpy.readers.mviri_l1b_fiduceo_nc.FiduceoMviriEasyFcdrFileHandler file_patterns: [ - 'FIDUCEO_FCDR_{level}_{sensor}_{platform}-{projection_longitude:f}_{start_time:%Y%m%d%H%M}_{end_time:%Y%m%d%H%M}_EASY_{processor_version}_{format_version}.nc' + 'FIDUCEO_FCDR_{level}_{sensor}_{platform}-{projection_longitude:f}_{start_time:%Y%m%d%H%M}_{end_time:%Y%m%d%H%M}_EASY_{processor_version}_{format_version}.nc', # Example: FIDUCEO_FCDR_L15_MVIRI_MET7-57.0_201701201000_201701201030_EASY_v2.6_fv3.1.nc + '{sensor}_FCDR-EASY_{level}_{platform}-E{projection_longitude:s}_{start_time:%Y%m%d%H%M}_{end_time:%Y%m%d%H%M}_{release}.nc' + # Example: MVIRI_FCDR-EASY_L15_MET7-E0000_200607060600_200607060630_0200.nc ] nc_full: file_reader: !!python/name:satpy.readers.mviri_l1b_fiduceo_nc.FiduceoMviriFullFcdrFileHandler file_patterns: [ - 'FIDUCEO_FCDR_{level}_{sensor}_{platform}-{projection_longitude:f}_{start_time:%Y%m%d%H%M}_{end_time:%Y%m%d%H%M}_FULL_{processor_version}_{format_version}.nc' + 'FIDUCEO_FCDR_{level}_{sensor}_{platform}-{projection_longitude:f}_{start_time:%Y%m%d%H%M}_{end_time:%Y%m%d%H%M}_FULL_{processor_version}_{format_version}.nc', # Example: FIDUCEO_FCDR_L15_MVIRI_MET7-57.0_201701201000_201701201030_FULL_v2.6_fv3.1.nc + '{sensor}_FCDR-FULL_{level}_{platform}-E{projection_longitude:s}_{start_time:%Y%m%d%H%M}_{end_time:%Y%m%d%H%M}_{release}.nc' + # Example: MVIRI_FCDR-FULL_L15_MET7-E0000_200607060600_200607060630_0200.nc ] datasets: diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index c170f87bd0..942d01fe71 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -464,7 +464,7 @@ def __init__(self, nc, decode_nc=True): def _decode_cf(self): # remove time before decoding and add again. time = self.get_time() - time_dims = self.nc["time_ir_wv"].dims + time_dims = self.nc[time.name].dims time = xr.where(time == time.attrs["_FillValue"], np.datetime64("NaT"), (time + time.attrs["add_offset"]).astype("datetime64[s]").astype("datetime64[ns]")) self.nc = self.nc.drop_vars(time.name) @@ -591,7 +591,12 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 # Projection longitude is not provided in the file, read it from the # filename. - self.projection_longitude = float(filename_info["projection_longitude"]) + if "." in str(filename_info["projection_longitude"]): + self.projection_longitude = float(filename_info["projection_longitude"]) + else: + self.projection_longitude = ( + float(filename_info["projection_longitude"][:2] + "." + filename_info["projection_longitude"][2:]) + ) self.calib_coefs = self._get_calib_coefs() self._get_angles = functools.lru_cache(maxsize=8)( diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 2cc7fdce61..b07b79740c 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -560,12 +560,15 @@ def test_file_pattern(self, reader): "FIDUCEO_FCDR_L15_MVIRI_MET7-57.0_201701201000_201701201030_FULL_v2.6_fv3.1.nc", "FIDUCEO_FCDR_L15_MVIRI_MET7-57.0_201701201000_201701201030_EASY_v2.6_fv3.1.nc", "FIDUCEO_FCDR_L15_MVIRI_MET7-00.0_201701201000_201701201030_EASY_v2.6_fv3.1.nc", + "MVIRI_FCDR-EASY_L15_MET7-E0000_200607060600_200607060630_0200.nc", + "MVIRI_FCDR-EASY_L15_MET7-E5700_200607060600_200607060630_0200.nc", + "MVIRI_FCDR-FULL_L15_MET7-E0000_200607060600_200607060630_0200.nc", "abcde", ] files = reader.select_files_from_pathnames(filenames) # only 3 out of 4 above should match - assert len(files) == 3 + assert len(files) == 6 class TestDatasetWrapper: From c1cd334b345fdf1d5f0c123fcf215ded16f421ef Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 3 Sep 2024 07:46:09 +0000 Subject: [PATCH 13/29] sync/merge with fork diffs --- satpy/readers/mviri_l1b_fiduceo_nc.py | 20 +++++++++++-------- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 4 ++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index 942d01fe71..1e096d6d67 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -452,24 +452,28 @@ def is_high_resol(resolution): class DatasetWrapper: """Helper class for accessing the dataset.""" - def __init__(self, nc, decode_nc=True): + def __init__(self, nc): """Wrap the given dataset.""" self.nc = nc - if decode_nc is True: - self._decode_cf() - self._fix_duplicate_dimensions(self.nc) - self.nc = self._chunk(self.nc) + self._decode_cf() + self._fix_duplicate_dimensions(self.nc) + self.nc = self._chunk(self.nc) def _decode_cf(self): # remove time before decoding and add again. + time_dims, time = self._decode_time() + self.nc = self.nc.drop_vars(time.name) + self.nc = xr.decode_cf(self.nc) + self.nc[time.name] = (time_dims, time.values) + + def _decode_time(self): time = self.get_time() time_dims = self.nc[time.name].dims time = xr.where(time == time.attrs["_FillValue"], np.datetime64("NaT"), (time + time.attrs["add_offset"]).astype("datetime64[s]").astype("datetime64[ns]")) - self.nc = self.nc.drop_vars(time.name) - self.nc = xr.decode_cf(self.nc) - self.nc[time.name] = (time_dims, time.values) + + return (time_dims, time) def _fix_duplicate_dimensions(self, nc): nc.variables["covariance_spectral_response_function_vis"].dims = ("srf_size_1", "srf_size_2") diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index b07b79740c..573cfa4e07 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -584,7 +584,7 @@ def test_fix_duplicate_dimensions(self): data_vars={"covariance_spectral_response_function_vis": (("srf_size", "srf_size"), [[1, 2], [3, 4]])} ) - foo_ds = DatasetWrapper(foo, decode_nc=False) + foo_ds = DatasetWrapper(foo) foo_ds._fix_duplicate_dimensions(foo_ds.nc) foo_exp = xr.Dataset( @@ -628,7 +628,7 @@ def test_reassign_coords(self): "x": [.3, .4] } ) - ds = DatasetWrapper(nc, decode_nc=False) + ds = DatasetWrapper(nc) foo = ds["foo"] xr.testing.assert_equal(foo, foo_exp) From abf916ed312df5e149c356be1e68bd3c0a3d4441 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 3 Sep 2024 09:29:16 +0000 Subject: [PATCH 14/29] adapt changes after xarray release 2024.7.0: include chunks with opening dataset; also rename dimension "channel" --- satpy/readers/mviri_l1b_fiduceo_nc.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index 1e096d6d67..16e300729b 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -162,7 +162,9 @@ from satpy.readers._geos_area import get_area_definition, get_area_extent, sampling_to_lfac_cfac from satpy.readers.file_handlers import BaseFileHandler +from satpy.utils import get_chunk_size_limit +CHUNK_SIZE = get_chunk_size_limit() EQUATOR_RADIUS = 6378140.0 POLE_RADIUS = 6356755.0 ALTITUDE = 42164000.0 - EQUATOR_RADIUS @@ -405,6 +407,7 @@ def interp_acq_time(time2d, target_y): """ # Compute mean timestamp per scanline time = time2d.mean(dim="x") + # If required, repeat timestamps in y-direction to obtain higher # resolution y = time.coords["y"].values @@ -458,7 +461,7 @@ def __init__(self, nc): self._decode_cf() self._fix_duplicate_dimensions(self.nc) - self.nc = self._chunk(self.nc) + def _decode_cf(self): # remove time before decoding and add again. @@ -478,17 +481,9 @@ def _decode_time(self): def _fix_duplicate_dimensions(self, nc): nc.variables["covariance_spectral_response_function_vis"].dims = ("srf_size_1", "srf_size_2") self.nc = nc.drop_dims("srf_size") - - def _chunk(self, nc): - - (chunk_size_y, chunk_size_x) = nc.variables["quality_pixel_bitmask"].encoding["chunksizes"] - chunks = { - "x": chunk_size_x, - "y": chunk_size_y, - "x_ir_wv": chunk_size_x, - "y_ir_wv": chunk_size_y - } - return nc.chunk(chunks) + nc.variables["channel_correlation_matrix_independent"].dims = ("channel_1", "channel_2") + nc.variables["channel_correlation_matrix_structured"].dims = ("channel_1", "channel_2") + self.nc = nc.drop_dims("channel") @property def attrs(self): @@ -586,6 +581,10 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 self.mask_bad_quality = mask_bad_quality nc_raw = xr.open_dataset( filename, + chunks={"x": CHUNK_SIZE, + "y": CHUNK_SIZE, + "x_ir_wv": CHUNK_SIZE, + "y_ir_wv": CHUNK_SIZE}, decode_cf=False, decode_times=False, mask_and_scale=False, From 5822e5045612334f4cf49882ebead6218e66f9b0 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 3 Sep 2024 13:39:56 +0000 Subject: [PATCH 15/29] removed chunks part from test_fix_duplicate_dimensions; adapted tests; moved decoding into separate method of DatasetWrapper --- satpy/readers/mviri_l1b_fiduceo_nc.py | 10 +++++-- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 29 ++++++++++--------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index 16e300729b..e989abe185 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -459,9 +459,6 @@ def __init__(self, nc): """Wrap the given dataset.""" self.nc = nc - self._decode_cf() - self._fix_duplicate_dimensions(self.nc) - def _decode_cf(self): # remove time before decoding and add again. @@ -535,6 +532,11 @@ def _cleanup_attrs(self, ds): # satpy warnings. ds.attrs.pop("ancillary_variables", None) + def prepare_input(self): + """Decode data and rename duplicate dimensions.""" + self._decode_cf() + self._fix_duplicate_dimensions(self.nc) + def get_time(self): """Get time coordinate. @@ -591,6 +593,8 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 ) self.nc = DatasetWrapper(nc_raw) + # decode times and fix duplicate dimensions + self.nc.prepare_input() # Projection longitude is not provided in the file, read it from the # filename. diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 573cfa4e07..ae7dd1e3f5 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -312,6 +312,8 @@ def fixture_fake_dataset(): "sub_satellite_latitude_start": np.nan, "sub_satellite_latitude_end": 0.1, "covariance_spectral_response_function_vis": (("srf_size", "srf_size"), cov), + "channel_correlation_matrix_independent": (("channel", "channel"), cov), + "channel_correlation_matrix_structured": (("channel", "channel"), cov) }, coords={ "y": [1, 2, 3, 4], @@ -319,8 +321,7 @@ def fixture_fake_dataset(): "y_ir_wv": [1, 2], "x_ir_wv": [1, 2], "y_tie": [1, 2], - "x_tie": [1, 2] - + "x_tie": [1, 2], }, attrs={"foo": "bar"} ) @@ -577,26 +578,28 @@ class TestDatasetWrapper: def test_fix_duplicate_dimensions(self): """Test the renaming of duplicate dimensions. - If duplicate dimensions are within the Dataset, opening the datasets with chunks throws an error. - Thus, the chunking needs to be done after opening the dataset and after the dimensions are renamed. + If duplicate dimensions are within the Dataset, opening the datasets with chunks throws a warning. + The dimensions need to be renamed. """ foo = xr.Dataset( - data_vars={"covariance_spectral_response_function_vis": - (("srf_size", "srf_size"), [[1, 2], [3, 4]])} + data_vars={ + "covariance_spectral_response_function_vis": (("srf_size", "srf_size"), [[1, 2], [3, 4]]), + "channel_correlation_matrix_independent": (("channel", "channel"), [[1, 2], [3, 4]]), + "channel_correlation_matrix_structured": (("channel", "channel"), [[1, 2], [3, 4]]) + } ) foo_ds = DatasetWrapper(foo) foo_ds._fix_duplicate_dimensions(foo_ds.nc) foo_exp = xr.Dataset( - data_vars={"covariance_spectral_response_function_vis": - (("srf_size_1", "srf_size_2"), [[1, 2], [3, 4]])} + data_vars={ + "covariance_spectral_response_function_vis": (("srf_size_1", "srf_size_2"), [[1, 2], [3, 4]]), + "channel_correlation_matrix_independent": (("channel_1", "channel_2"), [[1, 2], [3, 4]]), + "channel_correlation_matrix_structured": (("channel_1", "channel_2"), [[1, 2], [3, 4]]) + } ) - try: - foo_ds.nc.chunk("auto") - xr.testing.assert_allclose(foo_ds.nc, foo_exp) - except ValueError: - pytest.fail("Chunking failed.") + xr.testing.assert_allclose(foo_ds.nc, foo_exp) def test_reassign_coords(self): """Test reassigning of coordinates. From f41e06864bd6e5b4661b41a43a10804dffb6ae60 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Wed, 4 Sep 2024 08:45:03 +0000 Subject: [PATCH 16/29] moved code _get_projection_longitude() --- satpy/readers/mviri_l1b_fiduceo_nc.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index e989abe185..a313c92a6e 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -596,14 +596,8 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 # decode times and fix duplicate dimensions self.nc.prepare_input() - # Projection longitude is not provided in the file, read it from the - # filename. - if "." in str(filename_info["projection_longitude"]): - self.projection_longitude = float(filename_info["projection_longitude"]) - else: - self.projection_longitude = ( - float(filename_info["projection_longitude"][:2] + "." + filename_info["projection_longitude"][2:]) - ) + self.projection_longitude = self._get_projection_longitude(filename_info) + self.calib_coefs = self._get_calib_coefs() self._get_angles = functools.lru_cache(maxsize=8)( @@ -613,6 +607,13 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 self._get_acq_time_uncached ) + def _get_projection_longitude(self, filename_info): + """Read projection longitude from filename as it is not provided in the file.""" + if "." in str(filename_info["projection_longitude"]): + return float(filename_info["projection_longitude"]) + else: + return float(filename_info["projection_longitude"]) / 100 + def get_dataset(self, dataset_id, dataset_info): """Get the dataset.""" name = dataset_id["name"] From 6b079b1707b7ae242259f58bb6eacca14acf8208 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Wed, 4 Sep 2024 09:36:33 +0000 Subject: [PATCH 17/29] fix test_fix_duplicate_dimensions --- satpy/readers/mviri_l1b_fiduceo_nc.py | 52 +++++++++---------- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 13 +++-- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index a313c92a6e..e0297f7ec3 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -459,28 +459,11 @@ def __init__(self, nc): """Wrap the given dataset.""" self.nc = nc + # decode data + self._decode_cf() + # rename duplicate dimensions + self._fix_duplicate_dimensions(self.nc) - def _decode_cf(self): - # remove time before decoding and add again. - time_dims, time = self._decode_time() - self.nc = self.nc.drop_vars(time.name) - self.nc = xr.decode_cf(self.nc) - self.nc[time.name] = (time_dims, time.values) - - def _decode_time(self): - time = self.get_time() - time_dims = self.nc[time.name].dims - time = xr.where(time == time.attrs["_FillValue"], np.datetime64("NaT"), - (time + time.attrs["add_offset"]).astype("datetime64[s]").astype("datetime64[ns]")) - - return (time_dims, time) - - def _fix_duplicate_dimensions(self, nc): - nc.variables["covariance_spectral_response_function_vis"].dims = ("srf_size_1", "srf_size_2") - self.nc = nc.drop_dims("srf_size") - nc.variables["channel_correlation_matrix_independent"].dims = ("channel_1", "channel_2") - nc.variables["channel_correlation_matrix_structured"].dims = ("channel_1", "channel_2") - self.nc = nc.drop_dims("channel") @property def attrs(self): @@ -532,10 +515,27 @@ def _cleanup_attrs(self, ds): # satpy warnings. ds.attrs.pop("ancillary_variables", None) - def prepare_input(self): - """Decode data and rename duplicate dimensions.""" - self._decode_cf() - self._fix_duplicate_dimensions(self.nc) + def _decode_cf(self): + # remove time before decoding and add again. + time_dims, time = self._decode_time() + self.nc = self.nc.drop_vars(time.name) + self.nc = xr.decode_cf(self.nc) + self.nc[time.name] = (time_dims, time.values) + + def _decode_time(self): + time = self.get_time() + time_dims = self.nc[time.name].dims + time = xr.where(time == time.attrs["_FillValue"], np.datetime64("NaT"), + (time + time.attrs["add_offset"]).astype("datetime64[s]").astype("datetime64[ns]")) + + return (time_dims, time) + + def _fix_duplicate_dimensions(self, nc): + nc.variables["covariance_spectral_response_function_vis"].dims = ("srf_size_1", "srf_size_2") + self.nc = nc.drop_dims("srf_size") + nc.variables["channel_correlation_matrix_independent"].dims = ("channel_1", "channel_2") + nc.variables["channel_correlation_matrix_structured"].dims = ("channel_1", "channel_2") + self.nc = nc.drop_dims("channel") def get_time(self): """Get time coordinate. @@ -593,8 +593,6 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 ) self.nc = DatasetWrapper(nc_raw) - # decode times and fix duplicate dimensions - self.nc.prepare_input() self.projection_longitude = self._get_projection_longitude(filename_info) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index ae7dd1e3f5..9ced865eed 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -581,21 +581,26 @@ def test_fix_duplicate_dimensions(self): If duplicate dimensions are within the Dataset, opening the datasets with chunks throws a warning. The dimensions need to be renamed. """ + foo_time = 60*60 + foo_time_exp = np.datetime64("1970-01-01 01:00").astype("datetime64[ns]") + foo = xr.Dataset( data_vars={ "covariance_spectral_response_function_vis": (("srf_size", "srf_size"), [[1, 2], [3, 4]]), - "channel_correlation_matrix_independent": (("channel", "channel"), [[1, 2], [3, 4]]), - "channel_correlation_matrix_structured": (("channel", "channel"), [[1, 2], [3, 4]]) + "channel_correlation_matrix_independent": (("channel", "channel"), [[1, 2], [3, 4]]), + "channel_correlation_matrix_structured": (("channel", "channel"), [[1, 2], [3, 4]]), + "time_ir_wv": (("y_ir_wv", "x_ir_wv"), [[foo_time, foo_time], [foo_time, foo_time]], + {"_FillValue": 4294967295, "add_offset": 0}) } ) foo_ds = DatasetWrapper(foo) - foo_ds._fix_duplicate_dimensions(foo_ds.nc) foo_exp = xr.Dataset( data_vars={ "covariance_spectral_response_function_vis": (("srf_size_1", "srf_size_2"), [[1, 2], [3, 4]]), "channel_correlation_matrix_independent": (("channel_1", "channel_2"), [[1, 2], [3, 4]]), - "channel_correlation_matrix_structured": (("channel_1", "channel_2"), [[1, 2], [3, 4]]) + "channel_correlation_matrix_structured": (("channel_1", "channel_2"), [[1, 2], [3, 4]]), + "time_ir_wv": (("y_ir_wv", "x_ir_wv"), [[foo_time_exp, foo_time_exp], [foo_time_exp, foo_time_exp]]) } ) From fd099cd06797e7a89218a4ee831dede3f3c23667 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Mon, 16 Sep 2024 13:53:02 +0000 Subject: [PATCH 18/29] - suppress warnings for renaming duplicate dimensions - include u_independent_toa_bidirectional_reflectance in tests - patch decode_cf/fix_duplicate_dimensions in test_reassign_coords() --- satpy/readers/mviri_l1b_fiduceo_nc.py | 2 ++ .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index e0297f7ec3..b3e3ee275e 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -186,6 +186,8 @@ ] HIGH_RESOL = 2250 +warnings.filterwarnings("ignore", message="^.*We do not yet support duplicate dimension names, but " + "we do allow initial construction of the object.*$") class IRWVCalibrator: """Calibrate IR & WV channels.""" diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 9ced865eed..f5f8912ff8 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -127,6 +127,9 @@ }, attrs=attrs_exp ) + +u_struct_refl_exp = u_vis_refl_exp.copy() + acq_time_ir_wv_exp = [np.datetime64("NaT"), np.datetime64("1970-01-01 02:30").astype("datetime64[ns]")] wv_counts_exp = xr.DataArray( @@ -289,6 +292,7 @@ def fixture_fake_dataset(): "count_ir": (("y_ir_wv", "x_ir_wv"), count_ir), "toa_bidirectional_reflectance_vis": vis_refl_exp / 100, "u_independent_toa_bidirectional_reflectance": u_vis_refl_exp / 100, + "u_structured_toa_bidirectional_reflectance": u_vis_refl_exp / 100, "quality_pixel_bitmask": (("y", "x"), mask), "solar_zenith_angle": (("y_tie", "x_tie"), sza), "time_ir_wv": (("y_ir_wv", "x_ir_wv"), time), @@ -393,7 +397,8 @@ def test_init(self, file_handler): ("quality_pixel_bitmask", None, 2250, quality_pixel_bitmask_exp), ("solar_zenith_angle", None, 2250, sza_vis_exp), ("solar_zenith_angle", None, 4500, sza_ir_wv_exp), - ("u_independent_toa_bidirectional_reflectance", None, 4500, u_vis_refl_exp) + ("u_independent_toa_bidirectional_reflectance", None, 4500, u_vis_refl_exp), + ("u_structured_toa_bidirectional_reflectance", None, 4500, u_struct_refl_exp) ] ) def test_get_dataset(self, file_handler, name, calibration, resolution, @@ -636,9 +641,11 @@ def test_reassign_coords(self): "x": [.3, .4] } ) - ds = DatasetWrapper(nc) - foo = ds["foo"] - xr.testing.assert_equal(foo, foo_exp) + with mock.patch("satpy.readers.mviri_l1b_fiduceo_nc.DatasetWrapper._fix_duplicate_dimensions"): + with mock.patch("satpy.readers.mviri_l1b_fiduceo_nc.DatasetWrapper._decode_cf"): + ds = DatasetWrapper(nc) + foo = ds["foo"] + xr.testing.assert_equal(foo, foo_exp) class TestInterpolator: """Unit tests for Interpolator class.""" From 7c02d2d98618bbad33e9d8ffcd56dca6cd2274a8 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 17 Sep 2024 10:16:14 +0000 Subject: [PATCH 19/29] define fill_val in test --- satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index f5f8912ff8..b48e1f5046 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -45,6 +45,8 @@ # The following fixtures are not defined in this file, but are used and injected by Pytest: # - request +fill_val = int("4294967295") + attrs_exp: dict = { "platform": "MET7", "raw_metadata": {"foo": "bar"}, @@ -281,8 +283,8 @@ def fixture_fake_dataset(): cov = da.from_array([[1, 2], [3, 4]]) time = np.arange(4) * 60 * 60 - time[0] = 4294967295 - time[1] = 4294967295 + time[0] = fill_val + time[1] = fill_val time = time.reshape(2, 2) ds = xr.Dataset( @@ -332,7 +334,7 @@ def fixture_fake_dataset(): ds["count_ir"].attrs["ancillary_variables"] = "a_ir b_ir" ds["count_wv"].attrs["ancillary_variables"] = "a_wv b_wv" ds["quality_pixel_bitmask"].encoding["chunksizes"] = (2, 2) - ds["time_ir_wv"].attrs["_FillValue"] = 4294967295 + ds["time_ir_wv"].attrs["_FillValue"] = fill_val ds["time_ir_wv"].attrs["add_offset"] = 0 return ds @@ -595,7 +597,7 @@ def test_fix_duplicate_dimensions(self): "channel_correlation_matrix_independent": (("channel", "channel"), [[1, 2], [3, 4]]), "channel_correlation_matrix_structured": (("channel", "channel"), [[1, 2], [3, 4]]), "time_ir_wv": (("y_ir_wv", "x_ir_wv"), [[foo_time, foo_time], [foo_time, foo_time]], - {"_FillValue": 4294967295, "add_offset": 0}) + {"_FillValue": fill_val, "add_offset": 0}) } ) foo_ds = DatasetWrapper(foo) From e90d7d089813d213901f5e093bc934cd762cefeb Mon Sep 17 00:00:00 2001 From: bkremmli Date: Mon, 23 Sep 2024 13:40:17 +0000 Subject: [PATCH 20/29] ancillary changes like comments, etc. --- satpy/readers/mviri_l1b_fiduceo_nc.py | 12 +++++++----- .../tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 3 +-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/satpy/readers/mviri_l1b_fiduceo_nc.py b/satpy/readers/mviri_l1b_fiduceo_nc.py index b3e3ee275e..b11a7d07b1 100644 --- a/satpy/readers/mviri_l1b_fiduceo_nc.py +++ b/satpy/readers/mviri_l1b_fiduceo_nc.py @@ -461,9 +461,7 @@ def __init__(self, nc): """Wrap the given dataset.""" self.nc = nc - # decode data self._decode_cf() - # rename duplicate dimensions self._fix_duplicate_dimensions(self.nc) @@ -518,13 +516,15 @@ def _cleanup_attrs(self, ds): ds.attrs.pop("ancillary_variables", None) def _decode_cf(self): - # remove time before decoding and add again. + """Decode data.""" + # time decoding with decode_cf results in error - decode separately! time_dims, time = self._decode_time() self.nc = self.nc.drop_vars(time.name) self.nc = xr.decode_cf(self.nc) self.nc[time.name] = (time_dims, time.values) def _decode_time(self): + """Decode time using fill value and offset.""" time = self.get_time() time_dims = self.nc[time.name].dims time = xr.where(time == time.attrs["_FillValue"], np.datetime64("NaT"), @@ -533,6 +533,7 @@ def _decode_time(self): return (time_dims, time) def _fix_duplicate_dimensions(self, nc): + """Rename dimensions as duplicate dimensions names are not supported by xarray.""" nc.variables["covariance_spectral_response_function_vis"].dims = ("srf_size_1", "srf_size_2") self.nc = nc.drop_dims("srf_size") nc.variables["channel_correlation_matrix_independent"].dims = ("channel_1", "channel_2") @@ -589,6 +590,7 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 "y": CHUNK_SIZE, "x_ir_wv": CHUNK_SIZE, "y_ir_wv": CHUNK_SIZE}, + # see dataset wrapper for why decoding is disabled decode_cf=False, decode_times=False, mask_and_scale=False, @@ -611,8 +613,8 @@ def _get_projection_longitude(self, filename_info): """Read projection longitude from filename as it is not provided in the file.""" if "." in str(filename_info["projection_longitude"]): return float(filename_info["projection_longitude"]) - else: - return float(filename_info["projection_longitude"]) / 100 + + return float(filename_info["projection_longitude"]) / 100 def get_dataset(self, dataset_id, dataset_info): """Get the dataset.""" diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index b48e1f5046..172d34394f 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -45,7 +45,7 @@ # The following fixtures are not defined in this file, but are used and injected by Pytest: # - request -fill_val = int("4294967295") +fill_val = 4294967295 attrs_exp: dict = { "platform": "MET7", @@ -575,7 +575,6 @@ def test_file_pattern(self, reader): ] files = reader.select_files_from_pathnames(filenames) - # only 3 out of 4 above should match assert len(files) == 6 From 51d8e77a3b80441faf82e2077a86334c6aa768cc Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 24 Sep 2024 06:45:44 +0000 Subject: [PATCH 21/29] change fill_val definition in test, changes for Code Scene status --- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 47 ++++++++----------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 172d34394f..a3ef65ab07 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -28,7 +28,6 @@ import xarray as xr from pyproj import CRS from pyresample.geometry import AreaDefinition -from pytest_lazy_fixtures import lf as lazy_fixture from satpy.readers.mviri_l1b_fiduceo_nc import ( ALTITUDE, @@ -45,7 +44,7 @@ # The following fixtures are not defined in this file, but are used and injected by Pytest: # - request -fill_val = 4294967295 +fill_val = int("4294967295") attrs_exp: dict = { "platform": "MET7", @@ -257,6 +256,10 @@ height=2 ) +time_fake_dataset = np.arange(4) * 60 * 60 +time_fake_dataset[0] = fill_val +time_fake_dataset[1] = fill_val +time_fake_dataset = time_fake_dataset.reshape(2, 2) @pytest.fixture(name="fake_dataset") def fixture_fake_dataset(): @@ -266,26 +269,19 @@ def fixture_fake_dataset(): count_vis = da.linspace(0, 255, 16, dtype=np.uint8).reshape(4, 4) sza = da.from_array( np.array( - [[45, 90], - [0, 45]], + [[45, 90], [0, 45]], dtype=np.float32 ) ) mask = da.from_array( np.array( - [[0, 0, 0, 0], - [0, 0, 0, 0], - [0, 0, 1, 0], # 1 = "invalid" - [0, 0, 0, 0]], + [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 0]], # 1 = "invalid" dtype=np.uint8 ) ) cov = da.from_array([[1, 2], [3, 4]]) - time = np.arange(4) * 60 * 60 - time[0] = fill_val - time[1] = fill_val - time = time.reshape(2, 2) + time = time_fake_dataset ds = xr.Dataset( data_vars={ @@ -664,10 +660,10 @@ def fixture_time_ir_wv(self): coords={"y": [1, 3, 5, 7]} ) - @pytest.fixture(name="acq_time_vis_exp") - def fixture_acq_time_vis_exp(self): + @pytest.fixture(name="acq_time_exp") + def fixture_acq_time_exp(self): """Returns acq_time_vis_exp.""" - return xr.DataArray( + vis = xr.DataArray( [ np.datetime64("1970-01-01 01:30"), np.datetime64("1970-01-01 01:30"), @@ -682,10 +678,7 @@ def fixture_acq_time_vis_exp(self): coords={"y": [1, 2, 3, 4, 5, 6, 7, 8]} ) - @pytest.fixture(name="acq_time_ir_exp") - def fixture_acq_time_ir_exp(self): - """Returns acq_time_ir_exp.""" - return xr.DataArray( + ir = xr.DataArray( [ np.datetime64("1970-01-01 01:30"), np.datetime64("1970-01-01 03:30"), @@ -696,14 +689,12 @@ def fixture_acq_time_ir_exp(self): coords={"y": [1, 3, 5, 7]} ) - @pytest.mark.parametrize( - "acq_time_exp", - [ - lazy_fixture("acq_time_ir_exp"), - lazy_fixture("acq_time_vis_exp") - ] - ) + return vis, ir + def test_interp_acq_time(self, time_ir_wv, acq_time_exp): """Tests time interpolation.""" - res = Interpolator.interp_acq_time(time_ir_wv, target_y=acq_time_exp.coords["y"]) - xr.testing.assert_allclose(res, acq_time_exp) + res_vis = Interpolator.interp_acq_time(time_ir_wv, target_y=acq_time_exp[0].coords["y"]) + res_ir = Interpolator.interp_acq_time(time_ir_wv, target_y=acq_time_exp[1].coords["y"]) + + xr.testing.assert_allclose(res_vis, acq_time_exp[0]) + xr.testing.assert_allclose(res_ir, acq_time_exp[1]) From 89d9931fb51afa523f5aac4caec819f711db23ac Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 24 Sep 2024 08:04:29 +0000 Subject: [PATCH 22/29] change fill_val to int64 --- satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index a3ef65ab07..de6cec9913 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -44,7 +44,7 @@ # The following fixtures are not defined in this file, but are used and injected by Pytest: # - request -fill_val = int("4294967295") +fill_val = np.int64("4294967295") attrs_exp: dict = { "platform": "MET7", From 7d0ecefbf3b6981643a0f079af9f57e035aef6af Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 24 Sep 2024 08:15:55 +0000 Subject: [PATCH 23/29] include comments, create fixture_time_fake_dataset() --- .../reader_tests/test_mviri_l1b_fiduceo_nc.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index de6cec9913..fe03176713 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -44,7 +44,7 @@ # The following fixtures are not defined in this file, but are used and injected by Pytest: # - request -fill_val = np.int64("4294967295") +fill_val = np.int64("4294967295") # needs to be defined as int64 for windows attrs_exp: dict = { "platform": "MET7", @@ -256,13 +256,18 @@ height=2 ) -time_fake_dataset = np.arange(4) * 60 * 60 -time_fake_dataset[0] = fill_val -time_fake_dataset[1] = fill_val -time_fake_dataset = time_fake_dataset.reshape(2, 2) +@pytest.fixture(name="time_fake_dataset") +def fixture_time_fake_dataset(): + """Create time for fake dataset.""" + time = np.arange(4) * 60 * 60 + time[0] = fill_val + time[1] = fill_val + time = time.reshape(2, 2) + + return time @pytest.fixture(name="fake_dataset") -def fixture_fake_dataset(): +def fixture_fake_dataset(time_fake_dataset): """Create fake dataset.""" count_ir = da.linspace(0, 255, 4, dtype=np.uint8).reshape(2, 2) count_wv = da.linspace(0, 255, 4, dtype=np.uint8).reshape(2, 2) @@ -281,7 +286,6 @@ def fixture_fake_dataset(): ) cov = da.from_array([[1, 2], [3, 4]]) - time = time_fake_dataset ds = xr.Dataset( data_vars={ @@ -293,7 +297,7 @@ def fixture_fake_dataset(): "u_structured_toa_bidirectional_reflectance": u_vis_refl_exp / 100, "quality_pixel_bitmask": (("y", "x"), mask), "solar_zenith_angle": (("y_tie", "x_tie"), sza), - "time_ir_wv": (("y_ir_wv", "x_ir_wv"), time), + "time_ir_wv": (("y_ir_wv", "x_ir_wv"), time_fake_dataset), "a_ir": -5.0, "b_ir": 1.0, "bt_a_ir": 10.0, From 11cf080f0dc4bcaf26ea397ca10bcac4a9e0c5ac Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 24 Sep 2024 09:57:04 +0000 Subject: [PATCH 24/29] adapt fill_val definition --- satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index fe03176713..443996bd67 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -20,6 +20,7 @@ from __future__ import annotations import os +import sys from unittest import mock import dask.array as da @@ -44,7 +45,7 @@ # The following fixtures are not defined in this file, but are used and injected by Pytest: # - request -fill_val = np.int64("4294967295") # needs to be defined as int64 for windows +fill_val = min(sys.maxsize, 4294967295) # FillValue defined to be windows-compatible attrs_exp: dict = { "platform": "MET7", From 03ec1feacfcd1c631991ed741b05b2390b49f85d Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 24 Sep 2024 12:13:23 +0000 Subject: [PATCH 25/29] adapt fill_val definition --- satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 443996bd67..f18cde08c8 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -20,7 +20,6 @@ from __future__ import annotations import os -import sys from unittest import mock import dask.array as da @@ -45,7 +44,7 @@ # The following fixtures are not defined in this file, but are used and injected by Pytest: # - request -fill_val = min(sys.maxsize, 4294967295) # FillValue defined to be windows-compatible +fill_val = np.uint32(4294967295) # FillValue defined to be windows-compatible attrs_exp: dict = { "platform": "MET7", From bcae3f742eee4ea75b09aa90069a8cb0d6aad605 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Wed, 25 Sep 2024 13:58:40 +0000 Subject: [PATCH 26/29] lower fill_val for tests for windows compatibility --- satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index f18cde08c8..db3af4a0d6 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -44,7 +44,7 @@ # The following fixtures are not defined in this file, but are used and injected by Pytest: # - request -fill_val = np.uint32(4294967295) # FillValue defined to be windows-compatible +fill_val = np.uint32(429496729) # FillValue lower than in dataset to be windows-compatible attrs_exp: dict = { "platform": "MET7", From 38c8e9be8835ddf3259ee388118b2d628202cb51 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 1 Oct 2024 08:52:01 +0000 Subject: [PATCH 27/29] add test for get_projection_longitude, test original fill_val --- satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index db3af4a0d6..db30414d71 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -44,7 +44,7 @@ # The following fixtures are not defined in this file, but are used and injected by Pytest: # - request -fill_val = np.uint32(429496729) # FillValue lower than in dataset to be windows-compatible +fill_val = np.uint32(4294967295) # FillValue lower than in dataset to be windows-compatible attrs_exp: dict = { "platform": "MET7", @@ -345,7 +345,7 @@ def fixture_fake_dataset(time_fake_dataset): params=[FiduceoMviriEasyFcdrFileHandler, FiduceoMviriFullFcdrFileHandler] ) -def fixture_file_handler(fake_dataset, request): +def fixture_file_handler(fake_dataset, request, projection_longitude="57.0"): """Create mocked file handler.""" marker = request.node.get_closest_marker("file_handler_data") mask_bad_quality = True @@ -358,7 +358,7 @@ def fixture_file_handler(fake_dataset, request): filename="filename", filename_info={"platform": "MET7", "sensor": "MVIRI", - "projection_longitude": "57.0"}, + "projection_longitude": projection_longitude}, filetype_info={"foo": "bar"}, mask_bad_quality=mask_bad_quality ) @@ -379,7 +379,9 @@ def fixture_reader(): class TestFiduceoMviriFileHandlers: """Unit tests for FIDUCEO MVIRI file handlers.""" - def test_init(self, file_handler): + + @pytest.mark.parametrize("projection_longitude", ["57.0", "5700"]) + def test_init(self, file_handler, projection_longitude): """Test file handler initialization.""" assert file_handler.projection_longitude == 57.0 assert file_handler.mask_bad_quality is True From d88d3d3fed70c7c1c8fb03fc9b4d1ffe30fbc1e5 Mon Sep 17 00:00:00 2001 From: Stephan Finkensieper Date: Tue, 1 Oct 2024 09:45:46 +0000 Subject: [PATCH 28/29] Fix test miss --- satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index db30414d71..326a01963e 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -340,12 +340,18 @@ def fixture_fake_dataset(time_fake_dataset): return ds +@pytest.fixture(name="projection_longitude", params=["57.0"]) +def fixture_projection_longitude(request): + """Get projection longitude as string.""" + return request.param + + @pytest.fixture( name="file_handler", params=[FiduceoMviriEasyFcdrFileHandler, FiduceoMviriFullFcdrFileHandler] ) -def fixture_file_handler(fake_dataset, request, projection_longitude="57.0"): +def fixture_file_handler(fake_dataset, request, projection_longitude): """Create mocked file handler.""" marker = request.node.get_closest_marker("file_handler_data") mask_bad_quality = True @@ -380,7 +386,7 @@ class TestFiduceoMviriFileHandlers: """Unit tests for FIDUCEO MVIRI file handlers.""" - @pytest.mark.parametrize("projection_longitude", ["57.0", "5700"]) + @pytest.mark.parametrize("projection_longitude", ["57.0", "5700"], indirect=True) def test_init(self, file_handler, projection_longitude): """Test file handler initialization.""" assert file_handler.projection_longitude == 57.0 From 484b723235e317322d550d4e97867ecd0430f2b4 Mon Sep 17 00:00:00 2001 From: bkremmli Date: Tue, 1 Oct 2024 09:49:29 +0000 Subject: [PATCH 29/29] revert to smaller fill_val --- satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py index 326a01963e..ffc3e980e2 100644 --- a/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py +++ b/satpy/tests/reader_tests/test_mviri_l1b_fiduceo_nc.py @@ -44,7 +44,7 @@ # The following fixtures are not defined in this file, but are used and injected by Pytest: # - request -fill_val = np.uint32(4294967295) # FillValue lower than in dataset to be windows-compatible +fill_val = np.uint32(429496729) # FillValue lower than in dataset to be windows-compatible attrs_exp: dict = { "platform": "MET7",