From b20180a487ff967a2632b1a9d542439329b9d72d Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 7 Apr 2023 09:13:47 -0500 Subject: [PATCH 01/23] Create aliases for CLAVRx product names that are the bidirectional reflectance derived information for the L1b channels at 2 km. --- satpy/readers/clavrx.py | 47 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index 9565fa7169..cd5d0461e6 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -68,6 +68,23 @@ 'abi': 2004, } +CHANNEL_ALIASES = { + "abi": {"refl_0_47um_nom": {"name": "C01", "wavelength": 0.47}, + "refl_0_65um_nom": {"name": "C02", "wavelength": 0.64}, + "refl_0_86um_nom": {"name": "C03", "wavelength": 0.865}, + "refl_1_38um_nom": {"name": "C04", "wavelength": 1.378}, + "refl_1_60um_nom": {"name": "C05", "wavelength": 1.61}, + "refl_2_10um_nom": {"name": "C06", "wavelength": 2.25}, + }, + "ahi": {"refl_0_47um_nom": {"name": "C01", "wavelength": 0.47}, + "refl_0_55um_nom": {"name": "C02", "wavelength": 0.51}, + "refl_0_65um_nom": {"name": "C03", "wavelength": 0.64}, + "refl_0_86um_nom": {"name": "C04", "wavelength": 0.86}, + "refl_1_60um_nom": {"name": "C05", "wavelength": 1.61}, + "refl_2_10um_nom": {"name": "C06", "wavelength": 2.25} + }, +} + def _get_sensor(sensor: str) -> str: """Get the sensor.""" @@ -273,6 +290,19 @@ def get_metadata(sensor: str, platform: str, attrs: dict, ds_info: dict) -> dict return attr_info + @staticmethod + def _lookup_alias(vname: str, sensor: str, is_polar: bool) -> str: + """Return variable name if channel name is an alias for a different variable.""" + # Why? The aliases provide built-in access to the base sensor RGB composites. + if is_polar: + # Not implemented + pass + else: + dd = CHANNEL_ALIASES[sensor] + key = next(key for key, value in dd.items() if value["name"] == vname) + + return key + class CLAVRXHDF4FileHandler(HDF4FileHandler, _CLAVRxHelper): """A file handler for CLAVRx files.""" @@ -294,7 +324,7 @@ def end_time(self): return self.filename_info.get('end_time', self.start_time) def get_dataset(self, dataset_id, ds_info): - """Get a dataset.""" + """Get a dataset for Polar Sensors.""" var_name = ds_info.get('file_key', dataset_id['name']) data = self[var_name] data = _CLAVRxHelper._get_data(data, dataset_id) @@ -414,7 +444,8 @@ def _get_ds_info_for_data_arr(self, var_name): } return ds_info - def _is_2d_yx_data_array(self, data_arr): + @staticmethod + def _is_2d_yx_data_array(data_arr): has_y_dim = data_arr.dims[0] == "y" has_x_dim = data_arr.dims[1] == "x" return has_y_dim and has_x_dim @@ -435,6 +466,14 @@ def _available_new_datasets(self, handled_vars): ds_info = self._get_ds_info_for_data_arr(var_name) yield True, ds_info + alias_info = CHANNEL_ALIASES[self.sensor].get(var_name, None) + if alias_info is not None: + if self.nc.attrs["RESOLUTION_KM"] is not None: + alias_info["resolution"] = self.nc.attrs.get("RESOLUTION_KM", "2") + alias_info["resolution"] = alias_info["resolution"] * 1000. + ds_info.update(alias_info) + yield True, ds_info + def available_datasets(self, configured_datasets=None): """Dynamically discover what variables can be loaded from this file. @@ -470,8 +509,9 @@ def get_area_def(self, key): return _CLAVRxHelper._read_axi_fixed_grid(self.filename, l1b_att) def get_dataset(self, dataset_id, ds_info): - """Get a dataset.""" + """Get a dataset for supported geostationary sensors.""" var_name = ds_info.get('name', dataset_id['name']) + var_name = _CLAVRxHelper._lookup_alias(var_name, self.sensor, self._is_polar()) data = self[var_name] data = _CLAVRxHelper._get_data(data, dataset_id) data.attrs = _CLAVRxHelper.get_metadata(self.sensor, self.platform, @@ -480,5 +520,6 @@ def get_dataset(self, dataset_id, ds_info): def __getitem__(self, item): """Wrap around `self.nc[item]`.""" + # Check if 'item' is an alias: data = self.nc[item] return data From c6d0122761f4fdc0919a67db98fbb51b904dda54 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 11 Apr 2023 08:51:25 -0500 Subject: [PATCH 02/23] Update how alias is used with "file_key" in ds_info, which eliminates the need for a special lookup within reader Add tests to make sure aliases are created. --- satpy/readers/clavrx.py | 25 +++------- satpy/tests/reader_tests/test_clavrx_nc.py | 55 +++++++++++++--------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index cd5d0461e6..f8cb35cf51 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -290,19 +290,6 @@ def get_metadata(sensor: str, platform: str, attrs: dict, ds_info: dict) -> dict return attr_info - @staticmethod - def _lookup_alias(vname: str, sensor: str, is_polar: bool) -> str: - """Return variable name if channel name is an alias for a different variable.""" - # Why? The aliases provide built-in access to the base sensor RGB composites. - if is_polar: - # Not implemented - pass - else: - dd = CHANNEL_ALIASES[sensor] - key = next(key for key, value in dd.items() if value["name"] == vname) - - return key - class CLAVRXHDF4FileHandler(HDF4FileHandler, _CLAVRxHelper): """A file handler for CLAVRx files.""" @@ -464,13 +451,16 @@ def _available_new_datasets(self, handled_vars): continue ds_info = self._get_ds_info_for_data_arr(var_name) + ds_info.update({"file_key": var_name}) yield True, ds_info alias_info = CHANNEL_ALIASES[self.sensor].get(var_name, None) if alias_info is not None: - if self.nc.attrs["RESOLUTION_KM"] is not None: - alias_info["resolution"] = self.nc.attrs.get("RESOLUTION_KM", "2") - alias_info["resolution"] = alias_info["resolution"] * 1000. + alias_info.update({"file_key": var_name}) + if "RESOLUTION_KM" in self.nc.attrs: + alias_info["resolution"] = self.nc.attrs["RESOLUTION_KM"] * 1000. + else: + alias_info["resolution"] = NADIR_RESOLUTION[self.sensor] ds_info.update(alias_info) yield True, ds_info @@ -510,8 +500,7 @@ def get_area_def(self, key): def get_dataset(self, dataset_id, ds_info): """Get a dataset for supported geostationary sensors.""" - var_name = ds_info.get('name', dataset_id['name']) - var_name = _CLAVRxHelper._lookup_alias(var_name, self.sensor, self._is_polar()) + var_name = ds_info.get("file_key", dataset_id['name']) data = self[var_name] data = _CLAVRxHelper._get_data(data, dataset_id) data.attrs = _CLAVRxHelper.get_metadata(self.sensor, self.platform, diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx_nc.py index ea0dcaed9b..a72e46c354 100644 --- a/satpy/tests/reader_tests/test_clavrx_nc.py +++ b/satpy/tests/reader_tests/test_clavrx_nc.py @@ -37,6 +37,7 @@ DEFAULT_LON_DATA = np.linspace(5, 45, DEFAULT_FILE_SHAPE[1]).astype(DEFAULT_FILE_DTYPE) DEFAULT_LON_DATA = np.repeat([DEFAULT_LON_DATA], DEFAULT_FILE_SHAPE[0], axis=0) AHI_FILE = 'clavrx_H08_20210603_1500_B01_FLDK_R.level2.nc' +FILL_VALUE = -32768 def fake_test_content(filename, **kwargs): @@ -51,7 +52,8 @@ def fake_test_content(filename, **kwargs): longitude = xr.DataArray(DEFAULT_LON_DATA, dims=('scan_lines_along_track_direction', 'pixel_elements_along_scan_direction'), - attrs={'_FillValue': np.nan, + attrs={'_FillValue': -999., + 'SCALED': 0, 'scale_factor': 1., 'add_offset': 0., 'standard_name': 'longitude', @@ -61,37 +63,37 @@ def fake_test_content(filename, **kwargs): latitude = xr.DataArray(DEFAULT_LAT_DATA, dims=('scan_lines_along_track_direction', 'pixel_elements_along_scan_direction'), - attrs={'_FillValue': np.nan, + attrs={'_FillValue': -999., + 'SCALED': 0, 'scale_factor': 1., 'add_offset': 0., 'standard_name': 'latitude', 'units': 'degrees_south' }) - variable1 = xr.DataArray(DEFAULT_FILE_DATA.astype(np.float32), + variable1 = xr.DataArray(DEFAULT_FILE_DATA.astype(np.int8), dims=('scan_lines_along_track_direction', 'pixel_elements_along_scan_direction'), - attrs={'_FillValue': np.nan, - 'scale_factor': 1., - 'add_offset': 0., + attrs={'_FillValue': -127, + 'SCALED': 0, 'units': '1', - 'valid_range': [-32767, 32767], }) - # data with fill values - variable2 = xr.DataArray(DEFAULT_FILE_DATA.astype(np.float32), + # data with fill values and a file_type alias + variable2 = xr.DataArray(DEFAULT_FILE_DATA.astype(np.int16), dims=('scan_lines_along_track_direction', 'pixel_elements_along_scan_direction'), - attrs={'_FillValue': np.nan, - 'scale_factor': 1., - 'add_offset': 0., - 'units': '1', + attrs={'_FillValue': FILL_VALUE, + 'SCALED': 1, + 'scale_factor': 0.001861629, + 'add_offset': 59., + 'units': '%', 'valid_range': [-32767, 32767], }) - variable2 = variable2.where(variable2 % 2 != 0) + variable2 = variable2.where(variable2 % 2 != 0, FILL_VALUE) # category - variable3 = xr.DataArray(DEFAULT_FILE_FLAGS, + variable3 = xr.DataArray(DEFAULT_FILE_FLAGS.astype(np.int8), dims=('scan_lines_along_track_direction', 'pixel_elements_along_scan_direction'), attrs={'SCALED': 0, @@ -103,7 +105,7 @@ def fake_test_content(filename, **kwargs): 'longitude': longitude, 'latitude': latitude, 'variable1': variable1, - 'variable2': variable2, + 'refl_0_65um_nom': variable2, 'variable3': variable3 } @@ -141,7 +143,7 @@ def test_reader_creation(self, filenames, expected_loadables): @pytest.mark.parametrize( ("filenames", "expected_datasets"), - [([AHI_FILE], ['variable1', 'variable2', 'variable3']), ] + [([AHI_FILE], ['variable1', 'refl_0_65um_nom', 'variable3']), ] ) def test_available_datasets(self, filenames, expected_datasets): """Test that variables are dynamically discovered.""" @@ -154,10 +156,13 @@ def test_available_datasets(self, filenames, expected_datasets): avails = list(r.available_dataset_names) for var_name in expected_datasets: assert var_name in avails + # check extra datasets created by alias or coordinates + for var_name in ["latitude", "longitude", "C03"]: + assert var_name in avails @pytest.mark.parametrize( ("filenames", "loadable_ids"), - [([AHI_FILE], ['variable1', 'variable2', 'variable3']), ] + [([AHI_FILE], ['variable1', 'refl_0_65um_nom', 'C03', 'variable3']), ] ) def test_load_all_new_donor(self, filenames, loadable_ids): """Test loading all test datasets with new donor.""" @@ -184,18 +189,24 @@ def test_load_all_new_donor(self, filenames, loadable_ids): ) fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] datasets = r.load(loadable_ids) - assert len(datasets) == 3 + assert len(datasets) == 4 for v in datasets.values(): assert 'calibration' not in v.attrs - assert v.attrs['units'] == '1' + assert "units" in v.attrs assert isinstance(v.attrs['area'], AreaDefinition) assert v.attrs['platform_name'] == 'himawari8' assert v.attrs['sensor'] == 'ahi' assert 'rows_per_scan' not in v.coords.get('longitude').attrs - if v.attrs["name"] in ["variable1", "variable2"]: + if v.attrs["name"] == 'variable1': + assert "valid_range" not in v.attrs + assert v.dtype == np.float64 + assert "_FillValue" not in v.attrs + # should have file variable and one alias for reflectance + elif v.attrs["name"] in ["refl_0_65um_nom", "C03"]: assert isinstance(v.attrs["valid_range"], list) - assert v.dtype == np.float32 + assert v.dtype == np.float64 assert "_FillValue" not in v.attrs.keys() + assert (v.attrs["file_key"] == "refl_0_65um_nom") else: assert (datasets['variable3'].attrs.get('flag_meanings')) is not None assert (datasets['variable3'].attrs.get('flag_meanings') == '') From 87066011bb95e361c5b7b3cc15a621dfceb7a925 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 5 May 2023 17:12:46 -0500 Subject: [PATCH 03/23] Fixed is_polar() error improperly identifying ABI as polar. Updated tests to use fake goes data to test existing alias creation Reduced aliases to ABI and VIIRS channels --- satpy/etc/enhancements/generic.yaml | 23 ++++++ satpy/readers/clavrx.py | 81 +++++++++++----------- satpy/tests/reader_tests/test_clavrx.py | 2 +- satpy/tests/reader_tests/test_clavrx_nc.py | 26 +++---- 4 files changed, 79 insertions(+), 53 deletions(-) diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index ce1ce1bb94..45ae789dde 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -262,6 +262,29 @@ enhancements: stretch: linear cutoffs: [0.005, 0.005] + four_level_cloud_mask: + standard_name: cloud_mask + reader: clavrx + operations: + - name: palettize + method: !!python/name:satpy.enhancements.palettize + kwargs: + palettes: + - {'values': [-127,# Fill Value + 0, # Clear + 1, # Probably Clear + 2, # Probably Cloudy + 3, # Cloudy + ], + 'colors': [[ 0, 0, 0], # black,-127 = Fill Value + [ 94, 79, 162], # blue, 0 = Clear + [ 73, 228, 242], # cyan, 1 = Probably Clear + [158, 1, 66], # red, 2 = Probably Cloudy + [255, 255, 255], # white, 3 = Cloudy + ], + 'color_scale': 255, + } + sar-ice: standard_name: sar-ice operations: diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index f8cb35cf51..289adf6e0a 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -69,21 +69,18 @@ } CHANNEL_ALIASES = { - "abi": {"refl_0_47um_nom": {"name": "C01", "wavelength": 0.47}, - "refl_0_65um_nom": {"name": "C02", "wavelength": 0.64}, - "refl_0_86um_nom": {"name": "C03", "wavelength": 0.865}, - "refl_1_38um_nom": {"name": "C04", "wavelength": 1.378}, - "refl_1_60um_nom": {"name": "C05", "wavelength": 1.61}, - "refl_2_10um_nom": {"name": "C06", "wavelength": 2.25}, + "abi": {"refl_0_47um_nom": {"name": "C01", "wavelength": 0.47, "modifiers": ("sunz_corrected",)}, + "refl_0_65um_nom": {"name": "C02", "wavelength": 0.64, "modifiers": ("sunz_corrected",)}, + "refl_0_86um_nom": {"name": "C03", "wavelength": 0.865, "modifiers": ("sunz_corrected",)}, + "refl_1_38um_nom": {"name": "C04", "wavelength": 1.38, "modifiers": ("sunz_corrected",)}, + "refl_1_60um_nom": {"name": "C05", "wavelength": 1.61, "modifiers": ("sunz_corrected",)}, + "refl_2_10um_nom": {"name": "C06", "wavelength": 2.25, "modifiers": ("sunz_corrected",)}, }, - "ahi": {"refl_0_47um_nom": {"name": "C01", "wavelength": 0.47}, - "refl_0_55um_nom": {"name": "C02", "wavelength": 0.51}, - "refl_0_65um_nom": {"name": "C03", "wavelength": 0.64}, - "refl_0_86um_nom": {"name": "C04", "wavelength": 0.86}, - "refl_1_60um_nom": {"name": "C05", "wavelength": 1.61}, - "refl_2_10um_nom": {"name": "C06", "wavelength": 2.25} - }, -} + "viirs": {"refl_0_65um_nom": {"name": "I01", "wavelength": 0.64, "modifiers": ("sunz_corrected",)}, + "refl_1_38um_nom": {"name": "M09", "wavelength": 1.38, "modifiers": ("sunz_corrected",)}, + "refl_1_60um_nom": {"name": "I03", "wavelength": 1.61, "modifiers": ("sunz_corrected",)} + } + } def _get_sensor(sensor: str) -> str: @@ -143,8 +140,6 @@ def _get_data(data, dataset_id: dict) -> xr.DataArray: factor = attrs.pop('scale_factor', (np.ones(1, dtype=data.dtype))[0]) offset = attrs.pop('add_offset', (np.zeros(1, dtype=data.dtype))[0]) valid_range = attrs.get('valid_range', [None]) - if isinstance(valid_range, np.ndarray): - attrs["valid_range"] = valid_range.tolist() flags = not data.attrs.get("SCALED", 1) and any(data.attrs.get("flag_values", [None])) if not flags: @@ -152,15 +147,14 @@ def _get_data(data, dataset_id: dict) -> xr.DataArray: data = _CLAVRxHelper._scale_data(data, factor, offset) # don't need _FillValue if it has been applied. attrs.pop('_FillValue', None) - - if all(valid_range): - valid_min = _CLAVRxHelper._scale_data(valid_range[0], factor, offset) - valid_max = _CLAVRxHelper._scale_data(valid_range[1], factor, offset) - if flags: - data = data.where((data >= valid_min) & (data <= valid_max), fill) - else: + if isinstance(valid_range, np.ndarray): + valid_min = _CLAVRxHelper._scale_data(valid_range[0], factor, offset) + valid_max = _CLAVRxHelper._scale_data(valid_range[1], factor, offset) data = data.where((data >= valid_min) & (data <= valid_max)) - attrs['valid_range'] = [valid_min, valid_max] + else: + flag_values = attrs.get('flag_values', None) + if flag_values is not None and isinstance(flag_values, np.ndarray): + data = data.where((data >= flag_values[0]) & (data <= flag_values[-1]), fill) data.attrs = _CLAVRxHelper._remove_attributes(attrs) @@ -330,6 +324,15 @@ def get_nadir_resolution(self, sensor): elif res is not None: return int(res) + def _available_aliases(self, ds_info, current_var): + """Add alias if there is a match.""" + alias_info = CHANNEL_ALIASES.get(self.sensor).get(current_var, None) + if alias_info is not None: + alias_info.update({"file_key": current_var}) + alias_info["resolution"] = self.get_nadir_resolution(self.sensor) + ds_info.update(alias_info) + yield True, ds_info + def available_datasets(self, configured_datasets=None): """Automatically determine datasets provided by this file.""" self.sensor = _get_sensor(self.file_content.get('/attr/sensor')) @@ -375,6 +378,9 @@ def available_datasets(self, configured_datasets=None): ds_info['coordinates'] = ['longitude', 'latitude'] yield True, ds_info + if CHANNEL_ALIASES.get(self.sensor) is not None: + yield from self._available_aliases(ds_info, var_name) + def get_shape(self, dataset_id, ds_info): """Get the shape.""" var_name = ds_info.get('file_key', dataset_id['name']) @@ -425,11 +431,20 @@ def __init__(self, filename, filename_info, filetype_info): {"name": "longitude"}) def _get_ds_info_for_data_arr(self, var_name): + """Set data name and, if applicable, aliases.""" + channel_info = None ds_info = { 'file_type': self.filetype_info['file_type'], 'name': var_name, } - return ds_info + yield True, ds_info + + if CHANNEL_ALIASES.get(self.sensor) is not None: + channel_info = CHANNEL_ALIASES.get(self.sensor).get(var_name, None) + if channel_info is not None: + channel_info["file_key"] = var_name + ds_info.update(channel_info) + yield True, ds_info @staticmethod def _is_2d_yx_data_array(data_arr): @@ -450,19 +465,7 @@ def _available_new_datasets(self, handled_vars): # we need 'traditional' y/x dimensions currently continue - ds_info = self._get_ds_info_for_data_arr(var_name) - ds_info.update({"file_key": var_name}) - yield True, ds_info - - alias_info = CHANNEL_ALIASES[self.sensor].get(var_name, None) - if alias_info is not None: - alias_info.update({"file_key": var_name}) - if "RESOLUTION_KM" in self.nc.attrs: - alias_info["resolution"] = self.nc.attrs["RESOLUTION_KM"] * 1000. - else: - alias_info["resolution"] = NADIR_RESOLUTION[self.sensor] - ds_info.update(alias_info) - yield True, ds_info + yield from self._get_ds_info_for_data_arr(var_name) def available_datasets(self, configured_datasets=None): """Dynamically discover what variables can be loaded from this file. @@ -488,7 +491,7 @@ def _is_polar(self): l1b_att, inst_att = (str(self.nc.attrs.get('L1B', None)), str(self.nc.attrs.get('sensor', None))) - return (inst_att != 'AHI' and 'GOES' not in inst_att) or (l1b_att is None) + return (inst_att not in ['ABI', 'AHI'] and 'GOES' not in inst_att) or (l1b_att is None) def get_area_def(self, key): """Get the area definition of the data at hand.""" diff --git a/satpy/tests/reader_tests/test_clavrx.py b/satpy/tests/reader_tests/test_clavrx.py index 86e0cf1fa7..5a90bf873a 100644 --- a/satpy/tests/reader_tests/test_clavrx.py +++ b/satpy/tests/reader_tests/test_clavrx.py @@ -386,7 +386,7 @@ def test_load_all_old_donor(self): else: self.assertNotIn('_FillValue', v.attrs) if v.attrs["name"] == 'variable1': - self.assertIsInstance(v.attrs["valid_range"], list) + self.assertIsInstance(v.attrs["valid_range"], tuple) else: self.assertNotIn('valid_range', v.attrs) if 'flag_values' in v.attrs: diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx_nc.py index a72e46c354..5f0ba812b7 100644 --- a/satpy/tests/reader_tests/test_clavrx_nc.py +++ b/satpy/tests/reader_tests/test_clavrx_nc.py @@ -36,17 +36,17 @@ DEFAULT_LAT_DATA = np.repeat([DEFAULT_LAT_DATA], DEFAULT_FILE_SHAPE[0], axis=0) DEFAULT_LON_DATA = np.linspace(5, 45, DEFAULT_FILE_SHAPE[1]).astype(DEFAULT_FILE_DTYPE) DEFAULT_LON_DATA = np.repeat([DEFAULT_LON_DATA], DEFAULT_FILE_SHAPE[0], axis=0) -AHI_FILE = 'clavrx_H08_20210603_1500_B01_FLDK_R.level2.nc' +ABI_FILE = 'clavrx_OR_ABI-L1b-RadC-M6C01_G16_s20231021601173.level2.nc' FILL_VALUE = -32768 def fake_test_content(filename, **kwargs): """Mimic reader input file content.""" attrs = { - 'platform': 'HIM8', - 'sensor': 'AHI', + 'platform': 'G16', + 'sensor': 'ABI', # this is a Level 2 file that came from a L1B file - 'L1B': 'clavrx_H08_20210603_1500_B01_FLDK_R', + 'L1B': '"clavrx_OR_ABI-L1b-RadC-M6C01_G16_s20231021601173', } longitude = xr.DataArray(DEFAULT_LON_DATA, @@ -127,7 +127,7 @@ def setup_method(self): @pytest.mark.parametrize( ("filenames", "expected_loadables"), - [([AHI_FILE], 1)] + [([ABI_FILE], 1)] ) def test_reader_creation(self, filenames, expected_loadables): """Test basic initialization.""" @@ -143,7 +143,7 @@ def test_reader_creation(self, filenames, expected_loadables): @pytest.mark.parametrize( ("filenames", "expected_datasets"), - [([AHI_FILE], ['variable1', 'refl_0_65um_nom', 'variable3']), ] + [([ABI_FILE], ['variable1', 'refl_0_65um_nom', 'variable3']), ] ) def test_available_datasets(self, filenames, expected_datasets): """Test that variables are dynamically discovered.""" @@ -157,12 +157,12 @@ def test_available_datasets(self, filenames, expected_datasets): for var_name in expected_datasets: assert var_name in avails # check extra datasets created by alias or coordinates - for var_name in ["latitude", "longitude", "C03"]: + for var_name in ["latitude", "longitude"]: assert var_name in avails @pytest.mark.parametrize( ("filenames", "loadable_ids"), - [([AHI_FILE], ['variable1', 'refl_0_65um_nom', 'C03', 'variable3']), ] + [([ABI_FILE], ['variable1', 'refl_0_65um_nom', 'C02', 'variable3']), ] ) def test_load_all_new_donor(self, filenames, loadable_ids): """Test loading all test datasets with new donor.""" @@ -181,8 +181,8 @@ def test_load_all_new_donor(self, filenames, loadable_ids): semi_major_axis=6378137, semi_minor_axis=6356752.3142, perspective_point_height=35791000, - longitude_of_projection_origin=140.7, - sweep_angle_axis='y', + longitude_of_projection_origin=-137.2, + sweep_angle_axis='x', ) d.return_value = fake_donor = mock.MagicMock( variables={'goes_imager_projection': proj, 'x': x, 'y': y}, @@ -194,15 +194,15 @@ def test_load_all_new_donor(self, filenames, loadable_ids): assert 'calibration' not in v.attrs assert "units" in v.attrs assert isinstance(v.attrs['area'], AreaDefinition) - assert v.attrs['platform_name'] == 'himawari8' - assert v.attrs['sensor'] == 'ahi' + assert v.attrs['platform_name'] == 'GOES-16' + assert v.attrs['sensor'] == 'abi' assert 'rows_per_scan' not in v.coords.get('longitude').attrs if v.attrs["name"] == 'variable1': assert "valid_range" not in v.attrs assert v.dtype == np.float64 assert "_FillValue" not in v.attrs # should have file variable and one alias for reflectance - elif v.attrs["name"] in ["refl_0_65um_nom", "C03"]: + elif v.attrs["name"] in ["refl_0_65um_nom", "C02"]: assert isinstance(v.attrs["valid_range"], list) assert v.dtype == np.float64 assert "_FillValue" not in v.attrs.keys() From a48111e023e73fbabba1338901fdb9bfc5201d4a Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 9 May 2023 13:55:26 -0500 Subject: [PATCH 04/23] attempt to address complexity of available_datasets --- satpy/readers/clavrx.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index 289adf6e0a..d5154dca91 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -333,6 +333,23 @@ def _available_aliases(self, ds_info, current_var): ds_info.update(alias_info) yield True, ds_info + def _dynamic_datasets(self, nadir_resolution): + """Get data from file and build aliases.""" + # add new datasets + for var_name, val in self.file_content.items(): + if isinstance(val, SDS): + ds_info = { + 'file_type': self.filetype_info['file_type'], + 'resolution': nadir_resolution, + 'name': var_name, + } + if self._is_polar(): + ds_info['coordinates'] = ['longitude', 'latitude'] + yield True, ds_info + + if CHANNEL_ALIASES.get(self.sensor) is not None: + yield from self._available_aliases(ds_info, var_name) + def available_datasets(self, configured_datasets=None): """Automatically determine datasets provided by this file.""" self.sensor = _get_sensor(self.file_content.get('/attr/sensor')) @@ -366,20 +383,7 @@ def available_datasets(self, configured_datasets=None): # then we should keep it going down the chain yield is_avail, ds_info - # add new datasets - for var_name, val in self.file_content.items(): - if isinstance(val, SDS): - ds_info = { - 'file_type': self.filetype_info['file_type'], - 'resolution': nadir_resolution, - 'name': var_name, - } - if self._is_polar(): - ds_info['coordinates'] = ['longitude', 'latitude'] - yield True, ds_info - - if CHANNEL_ALIASES.get(self.sensor) is not None: - yield from self._available_aliases(ds_info, var_name) + yield from self._dynamic_datasets(nadir_resolution) def get_shape(self, dataset_id, ds_info): """Get the shape.""" @@ -452,7 +456,7 @@ def _is_2d_yx_data_array(data_arr): has_x_dim = data_arr.dims[1] == "x" return has_y_dim and has_x_dim - def _available_new_datasets(self, handled_vars): + def _available_file_datasets(self, handled_vars): """Metadata for available variables other than BT.""" possible_vars = list(self.nc.items()) + list(self.nc.coords.items()) for var_name, data_arr in possible_vars: @@ -485,7 +489,7 @@ def available_datasets(self, configured_datasets=None): if self.file_type_matches(ds_info['file_type']): handled_vars.add(ds_info['name']) yield self.file_type_matches(ds_info['file_type']), ds_info - yield from self._available_new_datasets(handled_vars) + yield from self._available_file_datasets(handled_vars) def _is_polar(self): l1b_att, inst_att = (str(self.nc.attrs.get('L1B', None)), From db8b011633cfbb3e1f1e6d8ff0bc8ba4400e82cc Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Mon, 15 May 2023 10:43:45 -0500 Subject: [PATCH 05/23] Trying to adjust for complexity error in available datasets --- satpy/readers/clavrx.py | 45 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index d5154dca91..3add1ea380 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -294,6 +294,9 @@ def __init__(self, filename, filename_info, filetype_info): filename_info, filetype_info) + self.sensor = _get_sensor(self.file_content.get('/attr/sensor')) + self.platform = _get_platform(self.file_content.get('/attr/platform')) + @property def start_time(self): """Get the start time.""" @@ -333,9 +336,26 @@ def _available_aliases(self, ds_info, current_var): ds_info.update(alias_info) yield True, ds_info + def _add_info_if_appropriate(self, is_avail, ds_info, nadir_resolution): + """Add more information if this reader can provide it.""" + new_info = ds_info.copy() # don't change input + this_res = ds_info.get('resolution') + var_name = ds_info.get('file_key', ds_info['name']) + matches = self.file_type_matches(ds_info['file_type']) + # we can confidently say that we can provide this dataset and can + # provide more info + if matches and var_name in self and this_res != nadir_resolution: + new_info['resolution'] = nadir_resolution + if self._is_polar(): + new_info['coordinates'] = ds_info.get('coordinates', ('longitude', 'latitude')) + yield True, new_info + elif is_avail is None: + # if we didn't know how to handle this dataset and no one else did + # then we should keep it going down the chain + yield is_avail, ds_info + def _dynamic_datasets(self, nadir_resolution): """Get data from file and build aliases.""" - # add new datasets for var_name, val in self.file_content.items(): if isinstance(val, SDS): ds_info = { @@ -352,36 +372,15 @@ def _dynamic_datasets(self, nadir_resolution): def available_datasets(self, configured_datasets=None): """Automatically determine datasets provided by this file.""" - self.sensor = _get_sensor(self.file_content.get('/attr/sensor')) - self.platform = _get_platform(self.file_content.get('/attr/platform')) - nadir_resolution = self.get_nadir_resolution(self.sensor) - coordinates = ('longitude', 'latitude') - handled_variables = set() # update previously configured datasets for is_avail, ds_info in (configured_datasets or []): - this_res = ds_info.get('resolution') - this_coords = ds_info.get('coordinates') # some other file handler knows how to load this if is_avail is not None: yield is_avail, ds_info - var_name = ds_info.get('file_key', ds_info['name']) - matches = self.file_type_matches(ds_info['file_type']) - # we can confidently say that we can provide this dataset and can - # provide more info - if matches and var_name in self and this_res != nadir_resolution: - handled_variables.add(var_name) - new_info = ds_info.copy() # don't mess up the above yielded - new_info['resolution'] = nadir_resolution - if self._is_polar() and this_coords is None: - new_info['coordinates'] = coordinates - yield True, new_info - elif is_avail is None: - # if we didn't know how to handle this dataset and no one else did - # then we should keep it going down the chain - yield is_avail, ds_info + yield from self._add_info_if_appropriate(is_avail, ds_info, nadir_resolution) yield from self._dynamic_datasets(nadir_resolution) From 01608a1bc0372a459d00587667ad241a2ed1a03d Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 19 May 2023 11:53:39 -0500 Subject: [PATCH 06/23] Add minor tests and refactor netcdf test to use a mock class so that repeating parameterize decorators can be removed. --- satpy/readers/clavrx.py | 70 +++---- satpy/tests/reader_tests/test_clavrx.py | 70 +++++-- satpy/tests/reader_tests/test_clavrx_nc.py | 218 ++++++++++++--------- 3 files changed, 215 insertions(+), 143 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index 3add1ea380..3d7455d209 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -296,6 +296,7 @@ def __init__(self, filename, filename_info, filetype_info): self.sensor = _get_sensor(self.file_content.get('/attr/sensor')) self.platform = _get_platform(self.file_content.get('/attr/platform')) + self.resolution = self.get_nadir_resolution(self.sensor) @property def start_time(self): @@ -329,60 +330,63 @@ def get_nadir_resolution(self, sensor): def _available_aliases(self, ds_info, current_var): """Add alias if there is a match.""" + new_info = ds_info.copy() alias_info = CHANNEL_ALIASES.get(self.sensor).get(current_var, None) if alias_info is not None: alias_info.update({"file_key": current_var}) - alias_info["resolution"] = self.get_nadir_resolution(self.sensor) - ds_info.update(alias_info) - yield True, ds_info + new_info.update(alias_info) + yield True, new_info - def _add_info_if_appropriate(self, is_avail, ds_info, nadir_resolution): + def _supplement_configured(self, configured_datasets=None): """Add more information if this reader can provide it.""" - new_info = ds_info.copy() # don't change input - this_res = ds_info.get('resolution') - var_name = ds_info.get('file_key', ds_info['name']) - matches = self.file_type_matches(ds_info['file_type']) - # we can confidently say that we can provide this dataset and can - # provide more info - if matches and var_name in self and this_res != nadir_resolution: - new_info['resolution'] = nadir_resolution - if self._is_polar(): - new_info['coordinates'] = ds_info.get('coordinates', ('longitude', 'latitude')) - yield True, new_info - elif is_avail is None: - # if we didn't know how to handle this dataset and no one else did - # then we should keep it going down the chain - yield is_avail, ds_info + for is_avail, ds_info in (configured_datasets or []): + # some other file handler knows how to load this + print(is_avail, ds_info) + if is_avail is not None: + yield is_avail, ds_info + + new_info = ds_info.copy() # don't change input + this_res = ds_info.get('resolution') + var_name = ds_info.get('file_key', ds_info['name']) + matches = self.file_type_matches(ds_info['file_type']) + # we can confidently say that we can provide this dataset and can + # provide more info + if matches and var_name in self and this_res != self.resolution: + new_info['resolution'] = self.resolution + if self._is_polar(): + new_info['coordinates'] = ds_info.get('coordinates', ('longitude', 'latitude')) + yield True, new_info + elif is_avail is None: + # if we didn't know how to handle this dataset and no one else did + # then we should keep it going down the chain + yield is_avail, ds_info - def _dynamic_datasets(self, nadir_resolution): + def _dynamic_datasets(self): """Get data from file and build aliases.""" for var_name, val in self.file_content.items(): if isinstance(val, SDS): ds_info = { 'file_type': self.filetype_info['file_type'], - 'resolution': nadir_resolution, + 'resolution': self.resolution, 'name': var_name, } if self._is_polar(): ds_info['coordinates'] = ['longitude', 'latitude'] - yield True, ds_info + # always yield what we have + yield True, ds_info if CHANNEL_ALIASES.get(self.sensor) is not None: + # yield variable as it is + # yield any associated aliases yield from self._available_aliases(ds_info, var_name) def available_datasets(self, configured_datasets=None): """Automatically determine datasets provided by this file.""" - nadir_resolution = self.get_nadir_resolution(self.sensor) - # update previously configured datasets - for is_avail, ds_info in (configured_datasets or []): - # some other file handler knows how to load this - if is_avail is not None: - yield is_avail, ds_info - - yield from self._add_info_if_appropriate(is_avail, ds_info, nadir_resolution) + yield from self._supplement_configured(configured_datasets) - yield from self._dynamic_datasets(nadir_resolution) + # get data from file dynamically + yield from self._dynamic_datasets() def get_shape(self, dataset_id, ds_info): """Get the shape.""" @@ -433,7 +437,7 @@ def __init__(self, filename, filename_info, filetype_info): self.nc.coords["longitude"] = _CLAVRxHelper._get_data(self.nc.coords["longitude"], {"name": "longitude"}) - def _get_ds_info_for_data_arr(self, var_name): + def _dynamic_dataset_info(self, var_name): """Set data name and, if applicable, aliases.""" channel_info = None ds_info = { @@ -468,7 +472,7 @@ def _available_file_datasets(self, handled_vars): # we need 'traditional' y/x dimensions currently continue - yield from self._get_ds_info_for_data_arr(var_name) + yield from self._dynamic_dataset_info(var_name) def available_datasets(self, configured_datasets=None): """Dynamically discover what variables can be loaded from this file. diff --git a/satpy/tests/reader_tests/test_clavrx.py b/satpy/tests/reader_tests/test_clavrx.py index 5a90bf873a..94a3da097f 100644 --- a/satpy/tests/reader_tests/test_clavrx.py +++ b/satpy/tests/reader_tests/test_clavrx.py @@ -48,7 +48,6 @@ def get_test_content(self, filename, filename_info, filetype_info): '/attr/platform': 'SNPP', '/attr/sensor': 'VIIRS', } - file_content['longitude'] = xr.DataArray( da.from_array(DEFAULT_LON_DATA, chunks=4096), attrs={ @@ -104,6 +103,20 @@ def get_test_content(self, filename, filename_info, filetype_info): }) file_content['variable3/shape'] = DEFAULT_FILE_SHAPE + file_content['refl_1_38um_nom'] = xr.DataArray( + da.from_array(DEFAULT_FILE_DATA, chunks=4096).astype(np.float32), + attrs={ + 'SCALED': 1, + 'add_offset': 59.0, + 'scale_factor': 0.0018616290763020515, + 'units': '%', + '_FillValue': -32768, + 'valid_range': [-32767, 32767], + 'actual_range': [-2., 120.], + 'actual_missing': -999.0 + }) + file_content['refl_1_38um_nom/shape'] = DEFAULT_FILE_SHAPE + return file_content @@ -204,6 +217,24 @@ def test_available_datasets(self): self.assertTrue(new_ds_infos[8][0]) self.assertEqual(new_ds_infos[8][1]['resolution'], 742) + def test_available_datasets_with_alias(self): + """Test availability of aliased dataset.""" + import xarray as xr + + from satpy.readers import load_reader + r = load_reader(self.reader_configs) + with mock.patch('satpy.readers.clavrx.SDS', xr.DataArray): + loadables = r.select_files_from_pathnames([ + 'clavrx_npp_d20170520_t2053581_e2055223_b28822.level2.hdf', + ]) + r.create_filehandlers(loadables) + available_ds = list(r.file_handlers['clavrx_hdf4'][0].available_datasets()) + + self.assertEqual(available_ds[5][1]["name"], "refl_1_38um_nom") + + self.assertEqual(available_ds[6][1]["name"], "M09") + self.assertEqual(available_ds[6][1]["file_key"], "refl_1_38um_nom") + def test_load_all(self): """Test loading all test datasets.""" import xarray as xr @@ -216,17 +247,17 @@ def test_load_all(self): ]) r.create_filehandlers(loadables) - var_list = ['variable1', 'variable2', 'variable3'] + var_list = ["M09", 'variable2', 'variable3'] datasets = r.load(var_list) self.assertEqual(len(datasets), len(var_list)) for v in datasets.values(): - self.assertEqual(v.attrs['units'], '1') + self.assertIn(v.attrs['units'], ['1', '%']) self.assertEqual(v.attrs['platform_name'], 'npp') self.assertEqual(v.attrs['sensor'], 'viirs') self.assertIsInstance(v.attrs['area'], SwathDefinition) self.assertEqual(v.attrs['area'].lons.attrs['rows_per_scan'], 16) self.assertEqual(v.attrs['area'].lats.attrs['rows_per_scan'], 16) - self.assertIsInstance(datasets["variable3"].attrs.get("flag_meanings"), list) + self.assertIsInstance(datasets["variable3"].attrs.get("flag_meanings"), list) class FakeHDF4FileHandlerGeo(FakeHDF4FileHandler): @@ -263,17 +294,20 @@ def get_test_content(self, filename, filename_info, filetype_info): }) file_content['latitude/shape'] = DEFAULT_FILE_SHAPE - file_content['variable1'] = xr.DataArray( + file_content['refl_1_38um_nom'] = xr.DataArray( DEFAULT_FILE_DATA.astype(np.float32), dims=('y', 'x'), attrs={ - '_FillValue': -1, - 'scale_factor': 1., - 'add_offset': 0., - 'units': '1', - 'valid_range': (-32767, 32767), + 'SCALED': 1, + 'add_offset': 59.0, + 'scale_factor': 0.0018616290763020515, + 'units': '%', + '_FillValue': -32768, + 'valid_range': [-32767, 32767], + 'actual_range': [-2., 120.], + 'actual_missing': -999.0 }) - file_content['variable1/shape'] = DEFAULT_FILE_SHAPE + file_content['refl_1_38um_nom/shape'] = DEFAULT_FILE_SHAPE # data with fill values file_content['variable2'] = xr.DataArray( @@ -347,7 +381,7 @@ def test_no_nav_donor(self): 'clavrx_H08_20180806_1800.level2.hdf', ]) r.create_filehandlers(loadables) - self.assertRaises(IOError, r.load, ['variable1', 'variable2', 'variable3']) + self.assertRaises(IOError, r.load, ['refl_1_38um_nom', 'variable2', 'variable3']) def test_load_all_old_donor(self): """Test loading all test datasets with old donor.""" @@ -375,18 +409,18 @@ def test_load_all_old_donor(self): variables={'Projection': proj, 'x': x, 'y': y}, ) fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] - datasets = r.load(['variable1', 'variable2', 'variable3']) + datasets = r.load(['refl_1_38um_nom', 'variable2', 'variable3']) self.assertEqual(len(datasets), 3) for v in datasets.values(): self.assertNotIn('calibration', v.attrs) - self.assertEqual(v.attrs['units'], '1') + self.assertIn(v.attrs['units'], ['1', '%']) self.assertIsInstance(v.attrs['area'], AreaDefinition) if v.attrs.get("flag_values"): self.assertIn('_FillValue', v.attrs) else: self.assertNotIn('_FillValue', v.attrs) - if v.attrs["name"] == 'variable1': - self.assertIsInstance(v.attrs["valid_range"], tuple) + if v.attrs["name"] == 'refl_1_38um_nom': + self.assertIsInstance(v.attrs["valid_range"], list) else: self.assertNotIn('valid_range', v.attrs) if 'flag_values' in v.attrs: @@ -419,11 +453,11 @@ def test_load_all_new_donor(self): variables={'goes_imager_projection': proj, 'x': x, 'y': y}, ) fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] - datasets = r.load(['variable1', 'variable2', 'variable3']) + datasets = r.load(['refl_1_38um_nom', 'variable2', 'variable3']) self.assertEqual(len(datasets), 3) for v in datasets.values(): self.assertNotIn('calibration', v.attrs) - self.assertEqual(v.attrs['units'], '1') + self.assertIn(v.attrs['units'], ['1', '%']) self.assertIsInstance(v.attrs['area'], AreaDefinition) self.assertTrue(v.attrs['area'].is_geostationary) self.assertEqual(v.attrs['platform_name'], 'himawari8') diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx_nc.py index 5f0ba812b7..a7dba879e5 100644 --- a/satpy/tests/reader_tests/test_clavrx_nc.py +++ b/satpy/tests/reader_tests/test_clavrx_nc.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2021 Satpy developers +# Copyright (c) 2018 Satpy developers # # This file is part of satpy. # @@ -18,13 +18,17 @@ """Module for testing the satpy.readers.clavrx module.""" import os +import unittest from unittest import mock import numpy as np -import pytest import xarray as xr from pyresample.geometry import AreaDefinition +from satpy.readers import load_reader +from satpy.tests.reader_tests.test_netCDF_utils import FakeNetCDF4FileHandler + +ABI_FILE = 'clavrx_OR_ABI-L1b-RadC-M6C01_G16_s20231021601173.level2.nc' DEFAULT_FILE_DTYPE = np.uint16 DEFAULT_FILE_SHAPE = (10, 300) DEFAULT_FILE_DATA = np.arange(DEFAULT_FILE_SHAPE[0] * DEFAULT_FILE_SHAPE[1], @@ -40,7 +44,7 @@ FILL_VALUE = -32768 -def fake_test_content(filename, **kwargs): +def fake_dataset(): """Mimic reader input file content.""" attrs = { 'platform': 'G16', @@ -115,99 +119,129 @@ def fake_test_content(filename, **kwargs): return ds -class TestCLAVRXReaderGeo: - """Test CLAVR-X Reader with Geo files.""" +class FakeNetCDF4FileHandlerCLAVRx(FakeNetCDF4FileHandler): + """Swap-in NetCDF4 File Handler.""" + + def get_test_content(self, filename, filename_info, filetype_info): + """Get a fake dataset.""" + return fake_dataset() + + +class TestCLAVRXReaderNetCDF(unittest.TestCase): + """Test CLAVR-X Reader with NetCDF files.""" yaml_file = "clavrx.yaml" + filename = ABI_FILE + loadable_ids = list(fake_dataset().keys()) - def setup_method(self): - """Read fake data.""" + def setUp(self): + """Wrap NetCDF file handler with a fake handler.""" from satpy._config import config_search_paths - self.reader_configs = config_search_paths(os.path.join('readers', self.yaml_file)) + from satpy.readers.clavrx import CLAVRXNetCDFFileHandler - @pytest.mark.parametrize( - ("filenames", "expected_loadables"), - [([ABI_FILE], 1)] - ) - def test_reader_creation(self, filenames, expected_loadables): - """Test basic initialization.""" - from satpy.readers import load_reader - with mock.patch('satpy.readers.clavrx.xr.open_dataset') as od: - od.side_effect = fake_test_content - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames(filenames) - assert len(loadables) == expected_loadables - r.create_filehandlers(loadables) - # make sure we have some files - assert r.file_handlers - - @pytest.mark.parametrize( - ("filenames", "expected_datasets"), - [([ABI_FILE], ['variable1', 'refl_0_65um_nom', 'variable3']), ] - ) - def test_available_datasets(self, filenames, expected_datasets): + self.reader_configs = config_search_paths(os.path.join('readers', self.yaml_file)) + # http://stackoverflow.com/questions/12219967/how-to-mock-a-base-class-with-python-mock-library + self.p = mock.patch.object(CLAVRXNetCDFFileHandler, '__bases__', + (FakeNetCDF4FileHandlerCLAVRx,), spec=True) + self.fake_open_dataset = mock.patch('satpy.readers.clavrx.xr.open_dataset', + return_value=fake_dataset()).start() + self.fake_handler = self.p.start() + self.p.is_local = True + + self.addCleanup(mock.patch.stopall) + + def test_init(self): + """Test basic init with no extra parameters.""" + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames([ABI_FILE]) + self.assertEqual(len(loadables), 1) + r.create_filehandlers(loadables) + # make sure we have some files + self.assertTrue(r.file_handlers) + + def test_available_datasets(self): """Test that variables are dynamically discovered.""" - from satpy.readers import load_reader - with mock.patch('satpy.readers.clavrx.xr.open_dataset') as od: - od.side_effect = fake_test_content - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames(filenames) - r.create_filehandlers(loadables) - avails = list(r.available_dataset_names) - for var_name in expected_datasets: - assert var_name in avails - # check extra datasets created by alias or coordinates - for var_name in ["latitude", "longitude"]: - assert var_name in avails - - @pytest.mark.parametrize( - ("filenames", "loadable_ids"), - [([ABI_FILE], ['variable1', 'refl_0_65um_nom', 'C02', 'variable3']), ] - ) - def test_load_all_new_donor(self, filenames, loadable_ids): + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames([ABI_FILE]) + r.create_filehandlers(loadables) + avails = list(r.available_dataset_names) + expected_datasets = self.loadable_ids + ["latitude", "longitude"] + self.assertEqual(avails.sort(), expected_datasets.sort()) + + def test_load_all_new_donor(self): """Test loading all test datasets with new donor.""" - from satpy.readers import load_reader - with mock.patch('satpy.readers.clavrx.xr.open_dataset') as od: - od.side_effect = fake_test_content - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames(filenames) - r.create_filehandlers(loadables) - with mock.patch('satpy.readers.clavrx.glob') as g, \ - mock.patch('satpy.readers.clavrx.netCDF4.Dataset') as d: - g.return_value = ['fake_donor.nc'] - x = np.linspace(-0.1518, 0.1518, 300) - y = np.linspace(0.1518, -0.1518, 10) - proj = mock.Mock( - semi_major_axis=6378137, - semi_minor_axis=6356752.3142, - perspective_point_height=35791000, - longitude_of_projection_origin=-137.2, - sweep_angle_axis='x', - ) - d.return_value = fake_donor = mock.MagicMock( - variables={'goes_imager_projection': proj, 'x': x, 'y': y}, - ) - fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] - datasets = r.load(loadable_ids) - assert len(datasets) == 4 - for v in datasets.values(): - assert 'calibration' not in v.attrs - assert "units" in v.attrs - assert isinstance(v.attrs['area'], AreaDefinition) - assert v.attrs['platform_name'] == 'GOES-16' - assert v.attrs['sensor'] == 'abi' - assert 'rows_per_scan' not in v.coords.get('longitude').attrs - if v.attrs["name"] == 'variable1': - assert "valid_range" not in v.attrs - assert v.dtype == np.float64 - assert "_FillValue" not in v.attrs - # should have file variable and one alias for reflectance - elif v.attrs["name"] in ["refl_0_65um_nom", "C02"]: - assert isinstance(v.attrs["valid_range"], list) - assert v.dtype == np.float64 - assert "_FillValue" not in v.attrs.keys() - assert (v.attrs["file_key"] == "refl_0_65um_nom") - else: - assert (datasets['variable3'].attrs.get('flag_meanings')) is not None - assert (datasets['variable3'].attrs.get('flag_meanings') == '') - assert np.issubdtype(v.dtype, np.integer) + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames([ABI_FILE]) + r.create_filehandlers(loadables) + with mock.patch('satpy.readers.clavrx.glob') as g, \ + mock.patch('satpy.readers.clavrx.netCDF4.Dataset') as d: + g.return_value = ['fake_donor.nc'] + x = np.linspace(-0.1518, 0.1518, 300) + y = np.linspace(0.1518, -0.1518, 10) + proj = mock.Mock( + semi_major_axis=6378137, + semi_minor_axis=6356752.3142, + perspective_point_height=35791000, + longitude_of_projection_origin=-137.2, + sweep_angle_axis='x', + ) + d.return_value = fake_donor = mock.MagicMock( + variables={'goes_imager_projection': proj, 'x': x, 'y': y}, + ) + fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] + + datasets = r.load(self.loadable_ids) + self.assertEqual(len(datasets), len(self.loadable_ids)) + for v in datasets.values(): + self.assertIsInstance(v.area, AreaDefinition) + self.assertEqual(v.platform_name, 'GOES-16') + self.assertEqual(v.sensor, 'abi') + + self.assertNotIn('calibration', v.attrs) + self.assertIn("units", v.attrs) + self.assertNotIn('rows_per_scan', v.coords.get('longitude').attrs) + # should have file variable and one alias for reflectance + if v.name == "variable1": + self.assertNotIn("valid_range", v.attrs) + self.assertNotIn("_FillValue", v.attrs) + self.assertEqual(np.float64, v.dtype) + elif v.name in ["refl_0_65um_nom", "C02"]: + self.assertIsInstance(v.valid_range, list) + self.assertEqual(np.float64, v.dtype) + self.assertNotIn("_FillValue", v.attrs) + if v.name == "C02": + self.assertEqual("refl_0_65um_nom", v.file_key) + else: + self.assertIsNotNone(datasets['variable3'].attrs.get('flag_meanings')) + self.assertEqual('', + datasets['variable3'].attrs.get('flag_meanings'), + ) + assert np.issubdtype(v.dtype, np.integer) + + def test_yaml_datasets(self): + """Test available_datasets with fake variables from YAML.""" + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames([ABI_FILE]) + r.create_filehandlers(loadables) + # mimic the YAML file being configured for more datasets + fake_dataset_info = [ + (None, {'name': 'yaml1', 'resolution': None, 'file_type': ['clavrx_nc']}), + (True, {'name': 'yaml2', 'resolution': 0.5, 'file_type': ['clavrx_nc']}), + ] + new_ds_infos = list(r.file_handlers['clavrx_nc'][0].available_datasets( + fake_dataset_info)) + self.assertEqual(len(new_ds_infos), 9) + + # we have this and can provide the resolution + self.assertTrue(new_ds_infos[0][0]) + self.assertEqual(new_ds_infos[0][1]['resolution'], 2004) # hardcoded + + # we have this, but previous file handler said it knew about it + # and it is producing the same resolution as what we have + self.assertTrue(new_ds_infos[1][0]) + self.assertEqual(new_ds_infos[1][1]['resolution'], 0.5) + + # we have this, but don't want to change the resolution + # because a previous handler said it has it + self.assertTrue(new_ds_infos[2][0]) + self.assertEqual(new_ds_infos[2][1]['resolution'], 2004) From f6caf710ba115118b1142e417eefe8d984cef85c Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 19 May 2023 12:23:20 -0500 Subject: [PATCH 07/23] Remove excessive if/then statements and actually test the alias name --- satpy/tests/reader_tests/test_clavrx_nc.py | 40 ++++++++++++---------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx_nc.py index a7dba879e5..b95a7dcce2 100644 --- a/satpy/tests/reader_tests/test_clavrx_nc.py +++ b/satpy/tests/reader_tests/test_clavrx_nc.py @@ -190,8 +190,27 @@ def test_load_all_new_donor(self): ) fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] - datasets = r.load(self.loadable_ids) - self.assertEqual(len(datasets), len(self.loadable_ids)) + datasets = r.load(self.loadable_ids + ["C02"]) + self.assertEqual(len(datasets), len(self.loadable_ids)+1) + + # should have file variable and one alias for reflectance + self.assertNotIn("valid_range", datasets["variable1"].attrs) + self.assertNotIn("_FillValue", datasets["variable1"].attrs) + self.assertEqual(np.float64, datasets["variable1"].dtype) + + assert np.issubdtype(datasets["variable3"].dtype, np.integer) + self.assertIsNotNone(datasets['variable3'].attrs.get('flag_meanings')) + self.assertEqual('', + datasets['variable3'].attrs.get('flag_meanings'), + ) + + self.assertIsInstance(datasets["refl_0_65um_nom"].valid_range, list) + self.assertEqual(np.float64, datasets["refl_0_65um_nom"].dtype) + self.assertNotIn("_FillValue", datasets["refl_0_65um_nom"].attrs) + + self.assertEqual("refl_0_65um_nom", datasets["C02"].file_key) + self.assertNotIn("_FillValue", datasets["C02"].attrs) + for v in datasets.values(): self.assertIsInstance(v.area, AreaDefinition) self.assertEqual(v.platform_name, 'GOES-16') @@ -200,23 +219,6 @@ def test_load_all_new_donor(self): self.assertNotIn('calibration', v.attrs) self.assertIn("units", v.attrs) self.assertNotIn('rows_per_scan', v.coords.get('longitude').attrs) - # should have file variable and one alias for reflectance - if v.name == "variable1": - self.assertNotIn("valid_range", v.attrs) - self.assertNotIn("_FillValue", v.attrs) - self.assertEqual(np.float64, v.dtype) - elif v.name in ["refl_0_65um_nom", "C02"]: - self.assertIsInstance(v.valid_range, list) - self.assertEqual(np.float64, v.dtype) - self.assertNotIn("_FillValue", v.attrs) - if v.name == "C02": - self.assertEqual("refl_0_65um_nom", v.file_key) - else: - self.assertIsNotNone(datasets['variable3'].attrs.get('flag_meanings')) - self.assertEqual('', - datasets['variable3'].attrs.get('flag_meanings'), - ) - assert np.issubdtype(v.dtype, np.integer) def test_yaml_datasets(self): """Test available_datasets with fake variables from YAML.""" From 26ad2b694a449434e67d7d41949b83d15e3804ba Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 19 May 2023 12:27:39 -0500 Subject: [PATCH 08/23] Remove extra space --- satpy/tests/reader_tests/test_clavrx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_clavrx.py b/satpy/tests/reader_tests/test_clavrx.py index 94a3da097f..7f1fecc2be 100644 --- a/satpy/tests/reader_tests/test_clavrx.py +++ b/satpy/tests/reader_tests/test_clavrx.py @@ -233,7 +233,7 @@ def test_available_datasets_with_alias(self): self.assertEqual(available_ds[5][1]["name"], "refl_1_38um_nom") self.assertEqual(available_ds[6][1]["name"], "M09") - self.assertEqual(available_ds[6][1]["file_key"], "refl_1_38um_nom") + self.assertEqual(available_ds[6][1]["file_key"], "refl_1_38um_nom") def test_load_all(self): """Test loading all test datasets.""" From d39a7cacd50e62dcb0c367aef54309c5e56e7133 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 26 May 2023 13:58:16 -0500 Subject: [PATCH 09/23] Fix mock cleanup error by using teardown rather than cleanup --- satpy/readers/clavrx.py | 116 +++++++++++---------- satpy/tests/reader_tests/test_clavrx.py | 1 + satpy/tests/reader_tests/test_clavrx_nc.py | 76 +++++++++++--- 3 files changed, 128 insertions(+), 65 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index 3d7455d209..f97c9cee19 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -33,7 +33,6 @@ LOG = logging.getLogger(__name__) - CF_UNITS = { 'none': '1', } @@ -69,18 +68,18 @@ } CHANNEL_ALIASES = { - "abi": {"refl_0_47um_nom": {"name": "C01", "wavelength": 0.47, "modifiers": ("sunz_corrected",)}, - "refl_0_65um_nom": {"name": "C02", "wavelength": 0.64, "modifiers": ("sunz_corrected",)}, - "refl_0_86um_nom": {"name": "C03", "wavelength": 0.865, "modifiers": ("sunz_corrected",)}, - "refl_1_38um_nom": {"name": "C04", "wavelength": 1.38, "modifiers": ("sunz_corrected",)}, - "refl_1_60um_nom": {"name": "C05", "wavelength": 1.61, "modifiers": ("sunz_corrected",)}, - "refl_2_10um_nom": {"name": "C06", "wavelength": 2.25, "modifiers": ("sunz_corrected",)}, - }, - "viirs": {"refl_0_65um_nom": {"name": "I01", "wavelength": 0.64, "modifiers": ("sunz_corrected",)}, - "refl_1_38um_nom": {"name": "M09", "wavelength": 1.38, "modifiers": ("sunz_corrected",)}, - "refl_1_60um_nom": {"name": "I03", "wavelength": 1.61, "modifiers": ("sunz_corrected",)} - } - } + "abi": {"refl_0_47um_nom": {"name": "C01", "wavelength": 0.47, "modifiers": ("sunz_corrected",)}, + "refl_0_65um_nom": {"name": "C02", "wavelength": 0.64, "modifiers": ("sunz_corrected",)}, + "refl_0_86um_nom": {"name": "C03", "wavelength": 0.865, "modifiers": ("sunz_corrected",)}, + "refl_1_38um_nom": {"name": "C04", "wavelength": 1.38, "modifiers": ("sunz_corrected",)}, + "refl_1_60um_nom": {"name": "C05", "wavelength": 1.61, "modifiers": ("sunz_corrected",)}, + "refl_2_10um_nom": {"name": "C06", "wavelength": 2.25, "modifiers": ("sunz_corrected",)}, + }, + "viirs": {"refl_0_65um_nom": {"name": "I01", "wavelength": 0.64, "modifiers": ("sunz_corrected",)}, + "refl_1_38um_nom": {"name": "M09", "wavelength": 1.38, "modifiers": ("sunz_corrected",)}, + "refl_1_60um_nom": {"name": "I03", "wavelength": 1.61, "modifiers": ("sunz_corrected",)} + } +} def _get_sensor(sensor: str) -> str: @@ -107,9 +106,29 @@ def _get_rows_per_scan(sensor: str) -> Optional[int]: return None +def _scale_data(data_arr: Union[xr.DataArray, int], scale_factor: float, add_offset: float) -> xr.DataArray: + """Scale data, if needed.""" + scaling_needed = not (scale_factor == 1.0 and add_offset == 0.0) + if scaling_needed: + data_arr = data_arr * scale_factor + add_offset + return data_arr + + class _CLAVRxHelper: """A base class for the CLAVRx File Handlers.""" + @staticmethod + def _get_nadir_resolution(sensor, resolution_from_filename_info): + """Get nadir resolution.""" + for k, v in NADIR_RESOLUTION.items(): + if sensor.startswith(k): + return v + res = resolution_from_filename_info + if res.endswith('m'): + return int(res[:-1]) + elif res is not None: + return int(res) + @staticmethod def _remove_attributes(attrs: dict) -> dict: """Remove attributes that described data before scaling.""" @@ -120,14 +139,6 @@ def _remove_attributes(attrs: dict) -> dict: attrs.pop(attr_key, None) return attrs - @staticmethod - def _scale_data(data_arr: Union[xr.DataArray, int], scale_factor: float, add_offset: float) -> xr.DataArray: - """Scale data, if needed.""" - scaling_needed = not (scale_factor == 1.0 and add_offset == 0.0) - if scaling_needed: - data_arr = data_arr * scale_factor + add_offset - return data_arr - @staticmethod def _get_data(data, dataset_id: dict) -> xr.DataArray: """Get a dataset.""" @@ -136,25 +147,28 @@ def _get_data(data, dataset_id: dict) -> xr.DataArray: attrs = data.attrs.copy() - fill = attrs.get('_FillValue') + # don't need these attributes after applied. factor = attrs.pop('scale_factor', (np.ones(1, dtype=data.dtype))[0]) offset = attrs.pop('add_offset', (np.zeros(1, dtype=data.dtype))[0]) + flag_values = data.attrs.get("flag_values", [None]) valid_range = attrs.get('valid_range', [None]) + if isinstance(valid_range, np.ndarray): + attrs["valid_range"] = valid_range.tolist() - flags = not data.attrs.get("SCALED", 1) and any(data.attrs.get("flag_values", [None])) - if not flags: + flags = not data.attrs.get("SCALED", 1) and any(flag_values) + if flags: + fill = attrs.get('_FillValue', None) + if isinstance(flag_values, np.ndarray) or isinstance(flag_values, list): + data = data.where((data >= flag_values[0]) & (data <= flag_values[-1]), fill) + else: + fill = attrs.pop('_FillValue', None) data = data.where(data != fill) - data = _CLAVRxHelper._scale_data(data, factor, offset) - # don't need _FillValue if it has been applied. - attrs.pop('_FillValue', None) - if isinstance(valid_range, np.ndarray): - valid_min = _CLAVRxHelper._scale_data(valid_range[0], factor, offset) - valid_max = _CLAVRxHelper._scale_data(valid_range[1], factor, offset) + data = _scale_data(data, factor, offset) + + if valid_range[0] is not None: + valid_min = _scale_data(valid_range[0], factor, offset) + valid_max = _scale_data(valid_range[1], factor, offset) data = data.where((data >= valid_min) & (data <= valid_max)) - else: - flag_values = attrs.get('flag_values', None) - if flag_values is not None and isinstance(flag_values, np.ndarray): - data = data.where((data >= flag_values[0]) & (data <= flag_values[-1]), fill) data.attrs = _CLAVRxHelper._remove_attributes(attrs) @@ -296,7 +310,8 @@ def __init__(self, filename, filename_info, filetype_info): self.sensor = _get_sensor(self.file_content.get('/attr/sensor')) self.platform = _get_platform(self.file_content.get('/attr/platform')) - self.resolution = self.get_nadir_resolution(self.sensor) + self.resolution = _CLAVRxHelper._get_nadir_resolution(self.sensor, + self.filename_info.get('resolution')) @property def start_time(self): @@ -317,17 +332,6 @@ def get_dataset(self, dataset_id, ds_info): data.attrs, ds_info) return data - def get_nadir_resolution(self, sensor): - """Get nadir resolution.""" - for k, v in NADIR_RESOLUTION.items(): - if sensor.startswith(k): - return v - res = self.filename_info.get('resolution') - if res.endswith('m'): - return int(res[:-1]) - elif res is not None: - return int(res) - def _available_aliases(self, ds_info, current_var): """Add alias if there is a match.""" new_info = ds_info.copy() @@ -341,7 +345,6 @@ def _supplement_configured(self, configured_datasets=None): """Add more information if this reader can provide it.""" for is_avail, ds_info in (configured_datasets or []): # some other file handler knows how to load this - print(is_avail, ds_info) if is_avail is not None: yield is_avail, ds_info @@ -431,6 +434,8 @@ def __init__(self, filename, filename_info, filetype_info): self.platform = _get_platform( self.filename_info.get('platform_shortname', None)) self.sensor = _get_sensor(self.nc.attrs.get('sensor', None)) + self.resolution = _CLAVRxHelper._get_nadir_resolution(self.sensor, + self.filename_info.get('resolution')) # coordinates need scaling and valid_range (mask_and_scale won't work on valid_range) self.nc.coords["latitude"] = _CLAVRxHelper._get_data(self.nc.coords["latitude"], {"name": "latitude"}) @@ -439,7 +444,6 @@ def __init__(self, filename, filename_info, filetype_info): def _dynamic_dataset_info(self, var_name): """Set data name and, if applicable, aliases.""" - channel_info = None ds_info = { 'file_type': self.filetype_info['file_type'], 'name': var_name, @@ -447,11 +451,12 @@ def _dynamic_dataset_info(self, var_name): yield True, ds_info if CHANNEL_ALIASES.get(self.sensor) is not None: + alias_info = ds_info.copy() channel_info = CHANNEL_ALIASES.get(self.sensor).get(var_name, None) if channel_info is not None: channel_info["file_key"] = var_name - ds_info.update(channel_info) - yield True, ds_info + alias_info.update(channel_info) + yield True, alias_info @staticmethod def _is_2d_yx_data_array(data_arr): @@ -488,10 +493,15 @@ def available_datasets(self, configured_datasets=None): # we don't know any more information than the previous # file handler so let's yield early yield is_avail, ds_info - continue - if self.file_type_matches(ds_info['file_type']): + + matches = self.file_type_matches(ds_info['file_type']) + if matches and ds_info.get('resolution') != self.resolution: + # reader knows something about this dataset (file type matches) + # add any information that this reader can add. + new_info = ds_info.copy() + new_info['resolution'] = self.resolution handled_vars.add(ds_info['name']) - yield self.file_type_matches(ds_info['file_type']), ds_info + yield True, new_info yield from self._available_file_datasets(handled_vars) def _is_polar(self): diff --git a/satpy/tests/reader_tests/test_clavrx.py b/satpy/tests/reader_tests/test_clavrx.py index 7f1fecc2be..f7c8f1f1cd 100644 --- a/satpy/tests/reader_tests/test_clavrx.py +++ b/satpy/tests/reader_tests/test_clavrx.py @@ -173,6 +173,7 @@ def test_available_datasets(self): (None, {'name': 'variable1', 'file_type': ['level_fake']}), (True, {'name': 'variable3', 'file_type': ['clavrx_hdf4']}), ] + new_ds_infos = list(r.file_handlers['clavrx_hdf4'][0].available_datasets( fake_dataset_info)) self.assertEqual(len(new_ds_infos), 9) diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx_nc.py index b95a7dcce2..0d3e6680cb 100644 --- a/satpy/tests/reader_tests/test_clavrx_nc.py +++ b/satpy/tests/reader_tests/test_clavrx_nc.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License along with # satpy. If not, see . """Module for testing the satpy.readers.clavrx module.""" - import os import unittest from unittest import mock @@ -30,11 +29,13 @@ ABI_FILE = 'clavrx_OR_ABI-L1b-RadC-M6C01_G16_s20231021601173.level2.nc' DEFAULT_FILE_DTYPE = np.uint16 -DEFAULT_FILE_SHAPE = (10, 300) +DEFAULT_FILE_SHAPE = (5, 5) DEFAULT_FILE_DATA = np.arange(DEFAULT_FILE_SHAPE[0] * DEFAULT_FILE_SHAPE[1], dtype=DEFAULT_FILE_DTYPE).reshape(DEFAULT_FILE_SHAPE) DEFAULT_FILE_FLAGS = np.arange(DEFAULT_FILE_SHAPE[0] * DEFAULT_FILE_SHAPE[1], dtype=np.byte).reshape(DEFAULT_FILE_SHAPE) +DEFAULT_FILE_FLAGS_BEYOND_FILL = DEFAULT_FILE_FLAGS +DEFAULT_FILE_FLAGS_BEYOND_FILL[-1][:-2] = [-127, -127, -128] DEFAULT_FILE_FACTORS = np.array([2.0, 1.0], dtype=np.float32) DEFAULT_LAT_DATA = np.linspace(45, 65, DEFAULT_FILE_SHAPE[1]).astype(DEFAULT_FILE_DTYPE) DEFAULT_LAT_DATA = np.repeat([DEFAULT_LAT_DATA], DEFAULT_FILE_SHAPE[0], axis=0) @@ -97,7 +98,7 @@ def fake_dataset(): variable2 = variable2.where(variable2 % 2 != 0, FILL_VALUE) # category - variable3 = xr.DataArray(DEFAULT_FILE_FLAGS.astype(np.int8), + var_flags = xr.DataArray(DEFAULT_FILE_FLAGS.astype(np.int8), dims=('scan_lines_along_track_direction', 'pixel_elements_along_scan_direction'), attrs={'SCALED': 0, @@ -105,12 +106,21 @@ def fake_dataset(): 'units': '1', 'flag_values': [0, 1, 2, 3]}) + out_of_range_flags = xr.DataArray(DEFAULT_FILE_FLAGS_BEYOND_FILL.astype(np.int8), + dims=('scan_lines_along_track_direction', + 'pixel_elements_along_scan_direction'), + attrs={'SCALED': 0, + '_FillValue': -127, + 'units': '1', + 'flag_values': [0, 1, 2, 3]}) + ds_vars = { 'longitude': longitude, 'latitude': latitude, 'variable1': variable1, 'refl_0_65um_nom': variable2, - 'variable3': variable3 + 'var_flags': var_flags, + 'out_of_range_flags': out_of_range_flags, } ds = xr.Dataset(ds_vars, attrs=attrs) @@ -142,13 +152,19 @@ def setUp(self): self.reader_configs = config_search_paths(os.path.join('readers', self.yaml_file)) # http://stackoverflow.com/questions/12219967/how-to-mock-a-base-class-with-python-mock-library self.p = mock.patch.object(CLAVRXNetCDFFileHandler, '__bases__', - (FakeNetCDF4FileHandlerCLAVRx,), spec=True) + (FakeNetCDF4FileHandlerCLAVRx,)) self.fake_open_dataset = mock.patch('satpy.readers.clavrx.xr.open_dataset', return_value=fake_dataset()).start() + self.expected_dataset = mock.patch('xarray.load_dataset', + return_value=fake_dataset()).start() self.fake_handler = self.p.start() self.p.is_local = True - self.addCleanup(mock.patch.stopall) + def tearDown(self): + """Stop wrapping the NetCDF4 file handler.""" + self.p.stop() + self.fake_open_dataset.stop() + self.expected_dataset.stop() def test_init(self): """Test basic init with no extra parameters.""" @@ -176,8 +192,8 @@ def test_load_all_new_donor(self): with mock.patch('satpy.readers.clavrx.glob') as g, \ mock.patch('satpy.readers.clavrx.netCDF4.Dataset') as d: g.return_value = ['fake_donor.nc'] - x = np.linspace(-0.1518, 0.1518, 300) - y = np.linspace(0.1518, -0.1518, 10) + x = np.linspace(-0.1518, 0.1518, 5) + y = np.linspace(0.1518, -0.1518, 5) proj = mock.Mock( semi_major_axis=6378137, semi_minor_axis=6356752.3142, @@ -198,11 +214,12 @@ def test_load_all_new_donor(self): self.assertNotIn("_FillValue", datasets["variable1"].attrs) self.assertEqual(np.float64, datasets["variable1"].dtype) - assert np.issubdtype(datasets["variable3"].dtype, np.integer) - self.assertIsNotNone(datasets['variable3'].attrs.get('flag_meanings')) + assert np.issubdtype(datasets["var_flags"].dtype, np.integer) + self.assertIsNotNone(datasets['var_flags'].attrs.get('flag_meanings')) self.assertEqual('', - datasets['variable3'].attrs.get('flag_meanings'), + datasets['var_flags'].attrs.get('flag_meanings'), ) + assert np.issubdtype(datasets["out_of_range_flags"].dtype, np.integer) self.assertIsInstance(datasets["refl_0_65um_nom"].valid_range, list) self.assertEqual(np.float64, datasets["refl_0_65um_nom"].dtype) @@ -232,7 +249,7 @@ def test_yaml_datasets(self): ] new_ds_infos = list(r.file_handlers['clavrx_nc'][0].available_datasets( fake_dataset_info)) - self.assertEqual(len(new_ds_infos), 9) + self.assertEqual(len(new_ds_infos), 10) # we have this and can provide the resolution self.assertTrue(new_ds_infos[0][0]) @@ -247,3 +264,38 @@ def test_yaml_datasets(self): # because a previous handler said it has it self.assertTrue(new_ds_infos[2][0]) self.assertEqual(new_ds_infos[2][1]['resolution'], 2004) + + def test_scale_data(self): + """Test that data is scaled when necessary and not scaled data are flags.""" + from satpy.readers.clavrx import _scale_data + """Test scale data and results.""" + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames([ABI_FILE]) + r.create_filehandlers(loadables) + with mock.patch('satpy.readers.clavrx.glob') as g, \ + mock.patch('satpy.readers.clavrx.netCDF4.Dataset') as d: + g.return_value = ['fake_donor.nc'] + x = np.linspace(-0.1518, 0.1518, 5) + y = np.linspace(0.1518, -0.1518, 5) + proj = mock.Mock( + semi_major_axis=6378137, + semi_minor_axis=6356752.3142, + perspective_point_height=35791000, + longitude_of_projection_origin=-137.2, + sweep_angle_axis='x', + ) + d.return_value = fake_donor = mock.MagicMock( + variables={'goes_imager_projection': proj, 'x': x, 'y': y}, + ) + fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] + + ds_scale = ["variable1", "refl_0_65um_nom"] + ds_no_scale = ["var_flags", "out_of_range_flags"] + + with mock.patch("satpy.readers.clavrx._scale_data", wraps=_scale_data) as scale_data: + r.load(ds_scale) + scale_data.assert_called() + + with mock.patch("satpy.readers.clavrx._scale_data", wraps=_scale_data) as scale_data2: + r.load(ds_no_scale) + scale_data2.assert_not_called() From 6433eb0efd361eca7ecdcb1361f6397ddefd6999 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 26 May 2023 15:22:14 -0500 Subject: [PATCH 10/23] Fix poor capitalization bug --- satpy/tests/reader_tests/test_clavrx_nc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx_nc.py index 0d3e6680cb..338d83cf0e 100644 --- a/satpy/tests/reader_tests/test_clavrx_nc.py +++ b/satpy/tests/reader_tests/test_clavrx_nc.py @@ -25,7 +25,7 @@ from pyresample.geometry import AreaDefinition from satpy.readers import load_reader -from satpy.tests.reader_tests.test_netCDF_utils import FakeNetCDF4FileHandler +from satpy.tests.reader_tests.test_netcdf_utils import FakeNetCDF4FileHandler ABI_FILE = 'clavrx_OR_ABI-L1b-RadC-M6C01_G16_s20231021601173.level2.nc' DEFAULT_FILE_DTYPE = np.uint16 From dace7ab5794e8df9464223fa71b9043704569d43 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sat, 27 May 2023 07:46:02 -0500 Subject: [PATCH 11/23] Fix donor name bug revealed when switching from AHI to ABI in tests Fix indent when creating info for alias --- satpy/readers/clavrx.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index f97c9cee19..1970bd641f 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -207,13 +207,17 @@ def _read_pug_fixed_grid(projection_coordinates: netCDF4.Variable, distance_mult return proj_dict @staticmethod - def _find_input_nc(filename: str, l1b_base: str) -> str: + def _find_input_nc(filename: str, sensor: str, l1b_base: str) -> str: dirname = os.path.dirname(filename) l1b_filename = os.path.join(dirname, l1b_base + '.nc') if os.path.exists(l1b_filename): return str(l1b_filename) - glob_pat = os.path.join(dirname, l1b_base + '*R20*.nc') + if sensor == "AHI": + glob_pat = os.path.join(dirname, l1b_base + '*R20*.nc') + else: + glob_pat = os.path.join(dirname, l1b_base + '*.nc') + LOG.debug("searching for {0}".format(glob_pat)) found_l1b_filenames = list(glob(glob_pat)) if len(found_l1b_filenames) == 0: @@ -223,7 +227,7 @@ def _find_input_nc(filename: str, l1b_base: str) -> str: return found_l1b_filenames[0] @staticmethod - def _read_axi_fixed_grid(filename: str, l1b_attr) -> geometry.AreaDefinition: + def _read_axi_fixed_grid(filename: str, sensor: str, l1b_attr) -> geometry.AreaDefinition: """Read a fixed grid. CLAVR-x does not transcribe fixed grid parameters to its output @@ -238,7 +242,7 @@ def _read_axi_fixed_grid(filename: str, l1b_attr) -> geometry.AreaDefinition: """ LOG.debug(f"looking for corresponding input file for {l1b_attr}" " to act as fixed grid navigation donor") - l1b_path = _CLAVRxHelper._find_input_nc(filename, l1b_attr) + l1b_path = _CLAVRxHelper._find_input_nc(filename, sensor, l1b_attr) LOG.info(f"CLAVR-x does not include fixed-grid parameters, use input file {l1b_path} as donor") l1b = netCDF4.Dataset(l1b_path) proj = None @@ -408,7 +412,7 @@ def get_area_def(self, key): return super(CLAVRXHDF4FileHandler, self).get_area_def(key) l1b_att = str(self.file_content.get('/attr/L1B', None)) - area_def = _CLAVRxHelper._read_axi_fixed_grid(self.filename, l1b_att) + area_def = _CLAVRxHelper._read_axi_fixed_grid(self.filename, self.sensor, l1b_att) return area_def @@ -453,10 +457,10 @@ def _dynamic_dataset_info(self, var_name): if CHANNEL_ALIASES.get(self.sensor) is not None: alias_info = ds_info.copy() channel_info = CHANNEL_ALIASES.get(self.sensor).get(var_name, None) - if channel_info is not None: - channel_info["file_key"] = var_name - alias_info.update(channel_info) - yield True, alias_info + if channel_info is not None: + channel_info["file_key"] = var_name + alias_info.update(channel_info) + yield True, alias_info @staticmethod def _is_2d_yx_data_array(data_arr): @@ -516,7 +520,7 @@ def get_area_def(self, key): return super(CLAVRXNetCDFFileHandler, self).get_area_def(key) l1b_att = str(self.nc.attrs.get('L1B', None)) - return _CLAVRxHelper._read_axi_fixed_grid(self.filename, l1b_att) + return _CLAVRxHelper._read_axi_fixed_grid(self.filename, self.sensor, l1b_att) def get_dataset(self, dataset_id, ds_info): """Get a dataset for supported geostationary sensors.""" From 6f685767aea14bd41394e8a5df17ace7a3ede571 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sat, 27 May 2023 07:48:12 -0500 Subject: [PATCH 12/23] Go back to using parameterize and add tests for alias. Clean up logic in some of the tests for yaml and expected datasets --- satpy/tests/reader_tests/test_clavrx_nc.py | 295 +++++++++++---------- 1 file changed, 157 insertions(+), 138 deletions(-) diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx_nc.py index 0d3e6680cb..0a6bdddfec 100644 --- a/satpy/tests/reader_tests/test_clavrx_nc.py +++ b/satpy/tests/reader_tests/test_clavrx_nc.py @@ -17,15 +17,14 @@ # satpy. If not, see . """Module for testing the satpy.readers.clavrx module.""" import os -import unittest from unittest import mock import numpy as np +import pytest import xarray as xr from pyresample.geometry import AreaDefinition from satpy.readers import load_reader -from satpy.tests.reader_tests.test_netCDF_utils import FakeNetCDF4FileHandler ABI_FILE = 'clavrx_OR_ABI-L1b-RadC-M6C01_G16_s20231021601173.level2.nc' DEFAULT_FILE_DTYPE = np.uint16 @@ -41,17 +40,18 @@ DEFAULT_LAT_DATA = np.repeat([DEFAULT_LAT_DATA], DEFAULT_FILE_SHAPE[0], axis=0) DEFAULT_LON_DATA = np.linspace(5, 45, DEFAULT_FILE_SHAPE[1]).astype(DEFAULT_FILE_DTYPE) DEFAULT_LON_DATA = np.repeat([DEFAULT_LON_DATA], DEFAULT_FILE_SHAPE[0], axis=0) -ABI_FILE = 'clavrx_OR_ABI-L1b-RadC-M6C01_G16_s20231021601173.level2.nc' +L1B_FILE = 'clavrx_OR_ABI-L1b-RadC-M6C01_G16_s20231021601173' +ABI_FILE = f'{L1B_FILE}.level2.nc' FILL_VALUE = -32768 -def fake_dataset(): +def fake_test_content(filename, **kwargs): """Mimic reader input file content.""" attrs = { 'platform': 'G16', 'sensor': 'ABI', # this is a Level 2 file that came from a L1B file - 'L1B': '"clavrx_OR_ABI-L1b-RadC-M6C01_G16_s20231021601173', + 'L1B': L1B_FILE, } longitude = xr.DataArray(DEFAULT_LON_DATA, @@ -129,149 +129,168 @@ def fake_dataset(): return ds -class FakeNetCDF4FileHandlerCLAVRx(FakeNetCDF4FileHandler): - """Swap-in NetCDF4 File Handler.""" - - def get_test_content(self, filename, filename_info, filetype_info): - """Get a fake dataset.""" - return fake_dataset() - - -class TestCLAVRXReaderNetCDF(unittest.TestCase): - """Test CLAVR-X Reader with NetCDF files.""" +class TestCLAVRXReaderGeo: + """Test CLAVR-X Reader with Geo files.""" yaml_file = "clavrx.yaml" - filename = ABI_FILE - loadable_ids = list(fake_dataset().keys()) - def setUp(self): - """Wrap NetCDF file handler with a fake handler.""" + def setup_method(self): + """Read fake data.""" from satpy._config import config_search_paths - from satpy.readers.clavrx import CLAVRXNetCDFFileHandler - self.reader_configs = config_search_paths(os.path.join('readers', self.yaml_file)) - # http://stackoverflow.com/questions/12219967/how-to-mock-a-base-class-with-python-mock-library - self.p = mock.patch.object(CLAVRXNetCDFFileHandler, '__bases__', - (FakeNetCDF4FileHandlerCLAVRx,)) - self.fake_open_dataset = mock.patch('satpy.readers.clavrx.xr.open_dataset', - return_value=fake_dataset()).start() - self.expected_dataset = mock.patch('xarray.load_dataset', - return_value=fake_dataset()).start() - self.fake_handler = self.p.start() - self.p.is_local = True - - def tearDown(self): - """Stop wrapping the NetCDF4 file handler.""" - self.p.stop() - self.fake_open_dataset.stop() - self.expected_dataset.stop() - - def test_init(self): - """Test basic init with no extra parameters.""" - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames([ABI_FILE]) - self.assertEqual(len(loadables), 1) - r.create_filehandlers(loadables) - # make sure we have some files - self.assertTrue(r.file_handlers) - - def test_available_datasets(self): + + @pytest.mark.parametrize( + ("filenames", "expected_loadables"), + [([ABI_FILE], 1)] + ) + def test_reader_creation(self, filenames, expected_loadables): + """Test basic initialization.""" + with mock.patch('satpy.readers.clavrx.xr.open_dataset') as od: + od.side_effect = fake_test_content + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames(filenames) + assert len(loadables) == expected_loadables + r.create_filehandlers(loadables) + # make sure we have some files + assert r.file_handlers + + @pytest.mark.parametrize( + ("filenames", "expected_datasets"), + [([ABI_FILE], ['variable1', 'refl_0_65um_nom', 'C02', 'var_flags', + 'out_of_range_flags', 'longitude', 'latitude']), ] + ) + def test_available_datasets(self, filenames, expected_datasets): """Test that variables are dynamically discovered.""" - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames([ABI_FILE]) - r.create_filehandlers(loadables) - avails = list(r.available_dataset_names) - expected_datasets = self.loadable_ids + ["latitude", "longitude"] - self.assertEqual(avails.sort(), expected_datasets.sort()) - - def test_load_all_new_donor(self): + from satpy.readers import load_reader + with mock.patch('satpy.readers.clavrx.xr.open_dataset') as od: + od.side_effect = fake_test_content + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames(filenames) + r.create_filehandlers(loadables) + avails = list(r.available_dataset_names) + for var_name in expected_datasets: + assert var_name in avails + + @pytest.mark.parametrize( + ("filenames", "loadable_ids"), + [([ABI_FILE], ['variable1', 'refl_0_65um_nom', 'var_flags', 'out_of_range_flags']), ] + ) + def test_load_all_new_donor(self, filenames, loadable_ids): """Test loading all test datasets with new donor.""" - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames([ABI_FILE]) - r.create_filehandlers(loadables) - with mock.patch('satpy.readers.clavrx.glob') as g, \ - mock.patch('satpy.readers.clavrx.netCDF4.Dataset') as d: - g.return_value = ['fake_donor.nc'] - x = np.linspace(-0.1518, 0.1518, 5) - y = np.linspace(0.1518, -0.1518, 5) - proj = mock.Mock( - semi_major_axis=6378137, - semi_minor_axis=6356752.3142, - perspective_point_height=35791000, - longitude_of_projection_origin=-137.2, - sweep_angle_axis='x', - ) - d.return_value = fake_donor = mock.MagicMock( - variables={'goes_imager_projection': proj, 'x': x, 'y': y}, - ) - fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] - - datasets = r.load(self.loadable_ids + ["C02"]) - self.assertEqual(len(datasets), len(self.loadable_ids)+1) - - # should have file variable and one alias for reflectance - self.assertNotIn("valid_range", datasets["variable1"].attrs) - self.assertNotIn("_FillValue", datasets["variable1"].attrs) - self.assertEqual(np.float64, datasets["variable1"].dtype) - - assert np.issubdtype(datasets["var_flags"].dtype, np.integer) - self.assertIsNotNone(datasets['var_flags'].attrs.get('flag_meanings')) - self.assertEqual('', - datasets['var_flags'].attrs.get('flag_meanings'), - ) - assert np.issubdtype(datasets["out_of_range_flags"].dtype, np.integer) - - self.assertIsInstance(datasets["refl_0_65um_nom"].valid_range, list) - self.assertEqual(np.float64, datasets["refl_0_65um_nom"].dtype) - self.assertNotIn("_FillValue", datasets["refl_0_65um_nom"].attrs) - - self.assertEqual("refl_0_65um_nom", datasets["C02"].file_key) - self.assertNotIn("_FillValue", datasets["C02"].attrs) - - for v in datasets.values(): - self.assertIsInstance(v.area, AreaDefinition) - self.assertEqual(v.platform_name, 'GOES-16') - self.assertEqual(v.sensor, 'abi') - - self.assertNotIn('calibration', v.attrs) - self.assertIn("units", v.attrs) - self.assertNotIn('rows_per_scan', v.coords.get('longitude').attrs) - - def test_yaml_datasets(self): + with mock.patch('satpy.readers.clavrx.xr.open_dataset') as od: + od.side_effect = fake_test_content + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames(filenames) + r.create_filehandlers(loadables) + with mock.patch('satpy.readers.clavrx.glob') as g, \ + mock.patch('satpy.readers.clavrx.netCDF4.Dataset') as d: + g.return_value = ['fake_donor.nc'] + x = np.linspace(-0.1518, 0.1518, DEFAULT_FILE_SHAPE[1]) + y = np.linspace(0.1518, -0.1518, DEFAULT_FILE_SHAPE[0]) + proj = mock.Mock( + semi_major_axis=6378137, + semi_minor_axis=6356752.3142, + perspective_point_height=35791000, + longitude_of_projection_origin=140.7, + sweep_angle_axis='y', + ) + d.return_value = fake_donor = mock.MagicMock( + variables={'goes_imager_projection': proj, 'x': x, 'y': y}, + ) + fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] + + datasets = r.load(loadable_ids + ["C02"]) + assert len(datasets) == len(loadable_ids)+1 + + # should have file variable and one alias for reflectance + assert "valid_range" not in datasets["variable1"].attrs + assert "_FillValue" not in datasets["variable1"].attrs + assert np.float64 == datasets["variable1"].dtype + + assert np.issubdtype(datasets["var_flags"].dtype, np.integer) + assert datasets['var_flags'].attrs.get('flag_meanings') is not None + assert '' == datasets['var_flags'].attrs.get('flag_meanings') + assert np.issubdtype(datasets["out_of_range_flags"].dtype, np.integer) + + assert isinstance(datasets["refl_0_65um_nom"].valid_range, list) + assert np.float64 == datasets["refl_0_65um_nom"].dtype + assert "_FillValue" not in datasets["refl_0_65um_nom"].attrs + + assert "refl_0_65um_nom" == datasets["C02"].file_key + assert "_FillValue" not in datasets["C02"].attrs + + for v in datasets.values(): + assert isinstance(v.area, AreaDefinition) + assert v.platform_name == 'GOES-16' + assert v.sensor == 'abi' + + assert 'calibration' not in v.attrs + assert 'rows_per_scan' not in v.coords.get('longitude').attrs + assert "units" in v.attrs + + @pytest.mark.parametrize( + ("filenames", "expected_loadables"), + [([ABI_FILE], 1)] + ) + def test_yaml_datasets(self, filenames, expected_loadables): """Test available_datasets with fake variables from YAML.""" - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames([ABI_FILE]) - r.create_filehandlers(loadables) - # mimic the YAML file being configured for more datasets - fake_dataset_info = [ - (None, {'name': 'yaml1', 'resolution': None, 'file_type': ['clavrx_nc']}), - (True, {'name': 'yaml2', 'resolution': 0.5, 'file_type': ['clavrx_nc']}), - ] - new_ds_infos = list(r.file_handlers['clavrx_nc'][0].available_datasets( - fake_dataset_info)) - self.assertEqual(len(new_ds_infos), 10) - - # we have this and can provide the resolution - self.assertTrue(new_ds_infos[0][0]) - self.assertEqual(new_ds_infos[0][1]['resolution'], 2004) # hardcoded - - # we have this, but previous file handler said it knew about it - # and it is producing the same resolution as what we have - self.assertTrue(new_ds_infos[1][0]) - self.assertEqual(new_ds_infos[1][1]['resolution'], 0.5) - - # we have this, but don't want to change the resolution - # because a previous handler said it has it - self.assertTrue(new_ds_infos[2][0]) - self.assertEqual(new_ds_infos[2][1]['resolution'], 2004) - - def test_scale_data(self): + with mock.patch('satpy.readers.clavrx.xr.open_dataset') as od: + od.side_effect = fake_test_content + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames(filenames) + r.create_filehandlers(loadables) + + with mock.patch('satpy.readers.clavrx.glob') as g, \ + mock.patch('satpy.readers.clavrx.netCDF4.Dataset') as d: + g.return_value = ['fake_donor.nc'] + x = np.linspace(-0.1518, 0.1518, 5) + y = np.linspace(0.1518, -0.1518, 5) + proj = mock.Mock( + semi_major_axis=6378137, + semi_minor_axis=6356752.3142, + perspective_point_height=35791000, + longitude_of_projection_origin=-137.2, + sweep_angle_axis='x', + ) + d.return_value = fake_donor = mock.MagicMock( + variables={'goes_imager_projection': proj, 'x': x, 'y': y}, + ) + fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] + # mimic the YAML file being configured for more datasets + fake_dataset_info = [ + (None, {'name': 'yaml1', 'resolution': None, 'file_type': ['clavrx_nc']}), + (True, {'name': 'yaml2', 'resolution': 0.5, 'file_type': ['clavrx_nc']}), + ] + new_ds_infos = list(r.file_handlers['clavrx_nc'][0].available_datasets( + fake_dataset_info)) + assert len(new_ds_infos) == 10 + + # we have this and can provide the resolution + assert (new_ds_infos[0][0]) + assert new_ds_infos[0][1]['resolution'] == 2004 # hardcoded + + # we have this, but previous file handler said it knew about it + # and it is producing the same resolution as what we have + assert (new_ds_infos[1][0]) + assert new_ds_infos[1][1]['resolution'] == 0.5 + + # we have this, but don't want to change the resolution + # because a previous handler said it has it + assert (new_ds_infos[2][0]) + assert new_ds_infos[2][1]['resolution'] == 2004 + + @pytest.mark.parametrize( + ("filenames", "loadable_ids"), + [([ABI_FILE], ['variable1', 'refl_0_65um_nom', 'var_flags', 'out_of_range_flags']), ] + ) + def test_scale_data(self, filenames, loadable_ids): """Test that data is scaled when necessary and not scaled data are flags.""" from satpy.readers.clavrx import _scale_data - """Test scale data and results.""" - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames([ABI_FILE]) - r.create_filehandlers(loadables) + with mock.patch('satpy.readers.clavrx.xr.open_dataset') as od: + od.side_effect = fake_test_content + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames(filenames) + r.create_filehandlers(loadables) with mock.patch('satpy.readers.clavrx.glob') as g, \ mock.patch('satpy.readers.clavrx.netCDF4.Dataset') as d: g.return_value = ['fake_donor.nc'] From 6a7e1c07f5f225fdc6e8bd76494e6c6da413b32e Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Mon, 5 Jun 2023 09:14:28 -0500 Subject: [PATCH 13/23] Trying to make sure valid range is given for non-flag variables (this is important for AWIPS) It also seems important that integer (flag data) does not have a valid range provided. Add tests to flag this problem if it appears again. --- satpy/readers/clavrx.py | 1 + satpy/tests/reader_tests/test_clavrx.py | 1 + satpy/tests/reader_tests/test_clavrx_nc.py | 3 +++ 3 files changed, 5 insertions(+) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index fdcd5ff8cf..23255adda9 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -171,6 +171,7 @@ def _get_data(data, dataset_id: dict) -> xr.DataArray: valid_min = _scale_data(valid_range[0], factor, offset) valid_max = _scale_data(valid_range[1], factor, offset) data = data.where((data >= valid_min) & (data <= valid_max)) + attrs['valid_range'] = [valid_min, valid_max] data.attrs = _CLAVRxHelper._remove_attributes(attrs) diff --git a/satpy/tests/reader_tests/test_clavrx.py b/satpy/tests/reader_tests/test_clavrx.py index f7c8f1f1cd..71d666b93d 100644 --- a/satpy/tests/reader_tests/test_clavrx.py +++ b/satpy/tests/reader_tests/test_clavrx.py @@ -421,6 +421,7 @@ def test_load_all_old_donor(self): else: self.assertNotIn('_FillValue', v.attrs) if v.attrs["name"] == 'refl_1_38um_nom': + self.assertIn("valid_range", v.attrs) self.assertIsInstance(v.attrs["valid_range"], list) else: self.assertNotIn('valid_range', v.attrs) diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx_nc.py index 0a6bdddfec..8487db48c7 100644 --- a/satpy/tests/reader_tests/test_clavrx_nc.py +++ b/satpy/tests/reader_tests/test_clavrx_nc.py @@ -206,15 +206,18 @@ def test_load_all_new_donor(self, filenames, loadable_ids): assert "valid_range" not in datasets["variable1"].attrs assert "_FillValue" not in datasets["variable1"].attrs assert np.float64 == datasets["variable1"].dtype + assert "valid_range" not in datasets["variable1"].attrs assert np.issubdtype(datasets["var_flags"].dtype, np.integer) assert datasets['var_flags'].attrs.get('flag_meanings') is not None assert '' == datasets['var_flags'].attrs.get('flag_meanings') assert np.issubdtype(datasets["out_of_range_flags"].dtype, np.integer) + assert "valid_range" not in datasets["out_of_range_flags"].attrs assert isinstance(datasets["refl_0_65um_nom"].valid_range, list) assert np.float64 == datasets["refl_0_65um_nom"].dtype assert "_FillValue" not in datasets["refl_0_65um_nom"].attrs + assert "valid_range" in datasets["refl_0_65um_nom"].attrs assert "refl_0_65um_nom" == datasets["C02"].file_key assert "_FillValue" not in datasets["C02"].attrs From 5c3cc8c3a80fd509ae6a4b1d51d532659b5f55b0 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Thu, 15 Jun 2023 13:46:44 -0500 Subject: [PATCH 14/23] BUG FIX: added back the handled variables which was lost when trying to reduce complexity. --- satpy/readers/clavrx.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index 23255adda9..39b3afc007 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -348,8 +348,9 @@ def _available_aliases(self, ds_info, current_var): new_info.update(alias_info) yield True, new_info - def _supplement_configured(self, configured_datasets=None): + def available_datasets(self, configured_datasets=None): """Add more information if this reader can provide it.""" + handled_variables = set() for is_avail, ds_info in (configured_datasets or []): # some other file handler knows how to load this if is_avail is not None: @@ -362,6 +363,7 @@ def _supplement_configured(self, configured_datasets=None): # we can confidently say that we can provide this dataset and can # provide more info if matches and var_name in self and this_res != self.resolution: + handled_variables.add(var_name) new_info['resolution'] = self.resolution if self._is_polar(): new_info['coordinates'] = ds_info.get('coordinates', ('longitude', 'latitude')) @@ -371,6 +373,9 @@ def _supplement_configured(self, configured_datasets=None): # then we should keep it going down the chain yield is_avail, ds_info + # get data from file dynamically + yield from self._dynamic_datasets() + def _dynamic_datasets(self): """Get data from file and build aliases.""" for var_name, val in self.file_content.items(): @@ -390,14 +395,6 @@ def _dynamic_datasets(self): # yield any associated aliases yield from self._available_aliases(ds_info, var_name) - def available_datasets(self, configured_datasets=None): - """Automatically determine datasets provided by this file.""" - # update previously configured datasets - yield from self._supplement_configured(configured_datasets) - - # get data from file dynamically - yield from self._dynamic_datasets() - def get_shape(self, dataset_id, ds_info): """Get the shape.""" var_name = ds_info.get('file_key', dataset_id['name']) From b68daa85c3c8713c7a2ebbf09dc0ab7d56fc05f8 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Wed, 17 Jan 2024 13:26:05 -0600 Subject: [PATCH 15/23] Add abi_geos to AreaDefinition so that the RGB images are named correctly. --- satpy/readers/clavrx.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index 39b3afc007..165de4e696 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -266,14 +266,24 @@ def _read_axi_fixed_grid(filename: str, sensor: str, l1b_attr) -> geometry.AreaD x, y = l1b['x'], l1b['y'] area_extent, ncols, nlines = _CLAVRxHelper._area_extent(x, y, h) - area = geometry.AreaDefinition( - 'ahi_geos', - "AHI L2 file area", - 'ahi_geos', - proj, - ncols, - nlines, - np.asarray(area_extent)) + if sensor == "abi": + area = geometry.AreaDefinition( + 'abi_geos', + "ABI L2 file area", + 'abi_geos', + proj, + ncols, + nlines, + np.asarray(area_extent)) + else: + area = geometry.AreaDefinition( + 'ahi_geos', + "AHI L2 file area", + 'ahi_geos', + proj, + ncols, + nlines, + np.asarray(area_extent)) return area From 81aaf034e9bf319ef1bd5eef5699494cf74e017c Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 19 Jan 2024 12:51:54 -0600 Subject: [PATCH 16/23] Fix problems with tests by splitting into separate files Check for a float32 because that is what satpy is expecting Update valid_range to list if np.ndarray to fix type error in scale_data. --- satpy/readers/clavrx.py | 3 +- .../tests/reader_tests/test_clavrx_geohdf.py | 246 ++++++++++++++++++ satpy/tests/reader_tests/test_clavrx_nc.py | 4 +- ...test_clavrx.py => test_clavrx_polarhdf.py} | 211 +-------------- 4 files changed, 251 insertions(+), 213 deletions(-) create mode 100644 satpy/tests/reader_tests/test_clavrx_geohdf.py rename satpy/tests/reader_tests/{test_clavrx.py => test_clavrx_polarhdf.py} (55%) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index 39ba70d03d..e23bde44e5 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -156,7 +156,8 @@ def _get_data(data, dataset_id: dict) -> xr.DataArray: valid_range = attrs.get("valid_range", [None]) if isinstance(valid_range, np.ndarray): - attrs["valid_range"] = valid_range.tolist() + valid_range = valid_range.tolist() + attrs["valid_range"] = valid_range flags = not data.attrs.get("SCALED", 1) and any(flag_values) if flags: diff --git a/satpy/tests/reader_tests/test_clavrx_geohdf.py b/satpy/tests/reader_tests/test_clavrx_geohdf.py new file mode 100644 index 0000000000..85a7f6faa3 --- /dev/null +++ b/satpy/tests/reader_tests/test_clavrx_geohdf.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Module for testing the satpy.readers.clavrx module.""" + +import os +import unittest +from unittest import mock + +import numpy as np +import pytest +import xarray as xr +from pyresample.geometry import AreaDefinition + +from satpy.tests.reader_tests.test_hdf4_utils import FakeHDF4FileHandler + +DEFAULT_FILE_DTYPE = np.uint16 +DEFAULT_FILE_SHAPE = (10, 300) +DEFAULT_FILE_DATA = np.arange(DEFAULT_FILE_SHAPE[0] * DEFAULT_FILE_SHAPE[1], + dtype=DEFAULT_FILE_DTYPE).reshape(DEFAULT_FILE_SHAPE) +DEFAULT_FILE_FACTORS = np.array([2.0, 1.0], dtype=np.float32) +DEFAULT_LAT_DATA = np.linspace(45, 65, DEFAULT_FILE_SHAPE[1]).astype(DEFAULT_FILE_DTYPE) +DEFAULT_LAT_DATA = np.repeat([DEFAULT_LAT_DATA], DEFAULT_FILE_SHAPE[0], axis=0) +DEFAULT_LON_DATA = np.linspace(5, 45, DEFAULT_FILE_SHAPE[1]).astype(DEFAULT_FILE_DTYPE) +DEFAULT_LON_DATA = np.repeat([DEFAULT_LON_DATA], DEFAULT_FILE_SHAPE[0], axis=0) + +class FakeHDF4FileHandlerGeo(FakeHDF4FileHandler): + """Swap-in HDF4 File Handler.""" + + def get_test_content(self, filename, filename_info, filetype_info): + """Mimic reader input file content.""" + file_content = { + "/attr/platform": "HIM8", + "/attr/sensor": "AHI", + # this is a Level 2 file that came from a L1B file + "/attr/L1B": "clavrx_H08_20180806_1800", + } + + file_content["longitude"] = xr.DataArray( + DEFAULT_LON_DATA, + dims=("y", "x"), + attrs={ + "_FillValue": np.nan, + "scale_factor": 1., + "add_offset": 0., + "standard_name": "longitude", + }) + file_content["longitude/shape"] = DEFAULT_FILE_SHAPE + + file_content["latitude"] = xr.DataArray( + DEFAULT_LAT_DATA, + dims=("y", "x"), + attrs={ + "_FillValue": np.nan, + "scale_factor": 1., + "add_offset": 0., + "standard_name": "latitude", + }) + file_content["latitude/shape"] = DEFAULT_FILE_SHAPE + + file_content["refl_1_38um_nom"] = xr.DataArray( + DEFAULT_FILE_DATA.astype(np.float32), + dims=("y", "x"), + attrs={ + "SCALED": 1, + "add_offset": 59.0, + "scale_factor": 0.0018616290763020515, + "units": "%", + "_FillValue": -32768, + "valid_range": [-32767, 32767], + "actual_range": [-2., 120.], + "actual_missing": -999.0 + }) + file_content["refl_1_38um_nom/shape"] = DEFAULT_FILE_SHAPE + + # data with fill values + file_content["variable2"] = xr.DataArray( + DEFAULT_FILE_DATA.astype(np.float32), + dims=("y", "x"), + attrs={ + "_FillValue": -1, + "scale_factor": 1., + "add_offset": 0., + "units": "1", + }) + file_content["variable2/shape"] = DEFAULT_FILE_SHAPE + file_content["variable2"] = file_content["variable2"].where( + file_content["variable2"] % 2 != 0) + + # category + file_content["variable3"] = xr.DataArray( + DEFAULT_FILE_DATA.astype(np.byte), + dims=("y", "x"), + attrs={ + "SCALED": 0, + "_FillValue": -128, + "flag_meanings": "clear water supercooled mixed ice unknown", + "flag_values": [0, 1, 2, 3, 4, 5], + "units": "1", + }) + file_content["variable3/shape"] = DEFAULT_FILE_SHAPE + + return file_content + + +class TestCLAVRXReaderGeo(unittest.TestCase): + """Test CLAVR-X Reader with Geo files.""" + + yaml_file = "clavrx.yaml" + + def setUp(self): + """Wrap HDF4 file handler with our own fake handler.""" + from satpy._config import config_search_paths + from satpy.readers.clavrx import CLAVRXHDF4FileHandler + self.reader_configs = config_search_paths(os.path.join("readers", self.yaml_file)) + # http://stackoverflow.com/questions/12219967/how-to-mock-a-base-class-with-python-mock-library + self.p = mock.patch.object(CLAVRXHDF4FileHandler, "__bases__", (FakeHDF4FileHandlerGeo,)) + self.fake_handler = self.p.start() + self.p.is_local = True + + def tearDown(self): + """Stop wrapping the NetCDF4 file handler.""" + self.p.stop() + + def test_init(self): + """Test basic init with no extra parameters.""" + from satpy.readers import load_reader + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames([ + "clavrx_H08_20180806_1800.level2.hdf", + ]) + assert len(loadables) == 1 + r.create_filehandlers(loadables) + # make sure we have some files + assert r.file_handlers + + def test_no_nav_donor(self): + """Test exception raised when no donor file is available.""" + import xarray as xr + + from satpy.readers import load_reader + r = load_reader(self.reader_configs) + fake_fn = "clavrx_H08_20180806_1800.level2.hdf" + with mock.patch("satpy.readers.clavrx.SDS", xr.DataArray): + loadables = r.select_files_from_pathnames([fake_fn]) + r.create_filehandlers(loadables) + l1b_base = fake_fn.split(".")[0] + msg = f"Missing navigation donor {l1b_base}" + with pytest.raises(IOError, match=msg): + r.load(["refl_1_38um_nom", "variable2", "variable3"]) + + def test_load_all_old_donor(self): + """Test loading all test datasets with old donor.""" + import xarray as xr + + from satpy.readers import load_reader + r = load_reader(self.reader_configs) + with mock.patch("satpy.readers.clavrx.SDS", xr.DataArray): + loadables = r.select_files_from_pathnames([ + "clavrx_H08_20180806_1800.level2.hdf", + ]) + r.create_filehandlers(loadables) + with mock.patch("satpy.readers.clavrx.glob") as g, mock.patch("satpy.readers.clavrx.netCDF4.Dataset") as d: + g.return_value = ["fake_donor.nc"] + x = np.linspace(-0.1518, 0.1518, 300) + y = np.linspace(0.1518, -0.1518, 10) + proj = mock.Mock( + semi_major_axis=6378.137, + semi_minor_axis=6356.7523142, + perspective_point_height=35791, + longitude_of_projection_origin=140.7, + sweep_angle_axis="y", + ) + d.return_value = fake_donor = mock.MagicMock( + variables={"Projection": proj, "x": x, "y": y}, + ) + fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] + datasets = r.load(["refl_1_38um_nom", "variable2", "variable3"]) + assert len(datasets) == 3 + for v in datasets.values(): + assert "calibration" not in v.attrs + assert v.attrs["units"] in ["1", "%"] + assert isinstance(v.attrs["area"], AreaDefinition) + if v.attrs.get("flag_values"): + assert "_FillValue" in v.attrs + else: + assert "_FillValue" not in v.attrs + if v.attrs["name"] == "refl_1_38um_nom": + assert "valid_range" in v.attrs + assert isinstance(v.attrs["valid_range"], list) + else: + assert "valid_range" not in v.attrs + if "flag_values" in v.attrs: + assert np.issubdtype(v.dtype, np.integer) + assert v.attrs.get("flag_meanings") is not None + + def test_load_all_new_donor(self): + """Test loading all test datasets with new donor.""" + import xarray as xr + + from satpy.readers import load_reader + r = load_reader(self.reader_configs) + with mock.patch("satpy.readers.clavrx.SDS", xr.DataArray): + loadables = r.select_files_from_pathnames([ + "clavrx_H08_20180806_1800.level2.hdf", + ]) + r.create_filehandlers(loadables) + with mock.patch("satpy.readers.clavrx.glob") as g, mock.patch("satpy.readers.clavrx.netCDF4.Dataset") as d: + g.return_value = ["fake_donor.nc"] + x = np.linspace(-0.1518, 0.1518, 300) + y = np.linspace(0.1518, -0.1518, 10) + proj = mock.Mock( + semi_major_axis=6378137, + semi_minor_axis=6356752.3142, + perspective_point_height=35791000, + longitude_of_projection_origin=140.7, + sweep_angle_axis="y", + ) + d.return_value = fake_donor = mock.MagicMock( + variables={"goes_imager_projection": proj, "x": x, "y": y}, + ) + fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] + datasets = r.load(["refl_1_38um_nom", "variable2", "variable3"]) + assert len(datasets) == 3 + for v in datasets.values(): + assert "calibration" not in v.attrs + assert v.attrs["units"] in ["1", "%"] + assert isinstance(v.attrs["area"], AreaDefinition) + assert v.attrs["area"].is_geostationary is True + assert v.attrs["platform_name"] == "himawari8" + assert v.attrs["sensor"] == "ahi" + assert datasets["variable3"].attrs.get("flag_meanings") is not None diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx_nc.py index 50b07306de..5f3e82f1dc 100644 --- a/satpy/tests/reader_tests/test_clavrx_nc.py +++ b/satpy/tests/reader_tests/test_clavrx_nc.py @@ -210,7 +210,7 @@ def test_load_all_new_donor(self, filenames, loadable_ids): # should have file variable and one alias for reflectance assert "valid_range" not in datasets["variable1"].attrs assert "_FillValue" not in datasets["variable1"].attrs - assert np.float64 == datasets["variable1"].dtype + assert np.float32 == datasets["variable1"].dtype assert "valid_range" not in datasets["variable1"].attrs assert np.issubdtype(datasets["var_flags"].dtype, np.integer) @@ -220,7 +220,7 @@ def test_load_all_new_donor(self, filenames, loadable_ids): assert "valid_range" not in datasets["out_of_range_flags"].attrs assert isinstance(datasets["refl_0_65um_nom"].valid_range, list) - assert np.float64 == datasets["refl_0_65um_nom"].dtype + assert np.float32 == datasets["refl_0_65um_nom"].dtype assert "_FillValue" not in datasets["refl_0_65um_nom"].attrs assert "valid_range" in datasets["refl_0_65um_nom"].attrs diff --git a/satpy/tests/reader_tests/test_clavrx.py b/satpy/tests/reader_tests/test_clavrx_polarhdf.py similarity index 55% rename from satpy/tests/reader_tests/test_clavrx.py rename to satpy/tests/reader_tests/test_clavrx_polarhdf.py index a962afacd2..6b69d8a923 100644 --- a/satpy/tests/reader_tests/test_clavrx.py +++ b/satpy/tests/reader_tests/test_clavrx_polarhdf.py @@ -23,9 +23,8 @@ import dask.array as da import numpy as np -import pytest import xarray as xr -from pyresample.geometry import AreaDefinition, SwathDefinition +from pyresample.geometry import SwathDefinition from satpy.tests.reader_tests.test_hdf4_utils import FakeHDF4FileHandler @@ -258,211 +257,3 @@ def test_load_all(self): assert v.attrs["area"].lons.attrs["rows_per_scan"] == 16 assert v.attrs["area"].lats.attrs["rows_per_scan"] == 16 assert isinstance(datasets["variable3"].attrs.get("flag_meanings"), list) - - -class FakeHDF4FileHandlerGeo(FakeHDF4FileHandler): - """Swap-in HDF4 File Handler.""" - - def get_test_content(self, filename, filename_info, filetype_info): - """Mimic reader input file content.""" - file_content = { - "/attr/platform": "HIM8", - "/attr/sensor": "AHI", - # this is a Level 2 file that came from a L1B file - "/attr/L1B": "clavrx_H08_20180806_1800", - } - - file_content["longitude"] = xr.DataArray( - DEFAULT_LON_DATA, - dims=("y", "x"), - attrs={ - "_FillValue": np.nan, - "scale_factor": 1., - "add_offset": 0., - "standard_name": "longitude", - }) - file_content["longitude/shape"] = DEFAULT_FILE_SHAPE - - file_content["latitude"] = xr.DataArray( - DEFAULT_LAT_DATA, - dims=("y", "x"), - attrs={ - "_FillValue": np.nan, - "scale_factor": 1., - "add_offset": 0., - "standard_name": "latitude", - }) - file_content["latitude/shape"] = DEFAULT_FILE_SHAPE - - file_content["refl_1_38um_nom"] = xr.DataArray( - DEFAULT_FILE_DATA.astype(np.float32), - dims=("y", "x"), - attrs={ - "SCALED": 1, - "add_offset": 59.0, - "scale_factor": 0.0018616290763020515, - "units": "%", - "_FillValue": -32768, - "valid_range": [-32767, 32767], - "actual_range": [-2., 120.], - "actual_missing": -999.0 - }) - file_content["refl_1_38um_nom/shape"] = DEFAULT_FILE_SHAPE - - # data with fill values - file_content["variable2"] = xr.DataArray( - DEFAULT_FILE_DATA.astype(np.float32), - dims=("y", "x"), - attrs={ - "_FillValue": -1, - "scale_factor": 1., - "add_offset": 0., - "units": "1", - }) - file_content["variable2/shape"] = DEFAULT_FILE_SHAPE - file_content["variable2"] = file_content["variable2"].where( - file_content["variable2"] % 2 != 0) - - # category - file_content["variable3"] = xr.DataArray( - DEFAULT_FILE_DATA.astype(np.byte), - dims=("y", "x"), - attrs={ - "SCALED": 0, - "_FillValue": -128, - "flag_meanings": "clear water supercooled mixed ice unknown", - "flag_values": [0, 1, 2, 3, 4, 5], - "units": "1", - }) - file_content["variable3/shape"] = DEFAULT_FILE_SHAPE - - return file_content - - -class TestCLAVRXReaderGeo(unittest.TestCase): - """Test CLAVR-X Reader with Geo files.""" - - yaml_file = "clavrx.yaml" - - def setUp(self): - """Wrap HDF4 file handler with our own fake handler.""" - from satpy._config import config_search_paths - from satpy.readers.clavrx import CLAVRXHDF4FileHandler - self.reader_configs = config_search_paths(os.path.join("readers", self.yaml_file)) - # http://stackoverflow.com/questions/12219967/how-to-mock-a-base-class-with-python-mock-library - self.p = mock.patch.object(CLAVRXHDF4FileHandler, "__bases__", (FakeHDF4FileHandlerGeo,)) - self.fake_handler = self.p.start() - self.p.is_local = True - - def tearDown(self): - """Stop wrapping the NetCDF4 file handler.""" - self.p.stop() - - def test_init(self): - """Test basic init with no extra parameters.""" - from satpy.readers import load_reader - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames([ - "clavrx_H08_20180806_1800.level2.hdf", - ]) - assert len(loadables) == 1 - r.create_filehandlers(loadables) - # make sure we have some files - assert r.file_handlers - - def test_no_nav_donor(self): - """Test exception raised when no donor file is available.""" - import xarray as xr - - from satpy.readers import load_reader - r = load_reader(self.reader_configs) - fake_fn = "clavrx_H08_20180806_1800.level2.hdf" - with mock.patch("satpy.readers.clavrx.SDS", xr.DataArray): - loadables = r.select_files_from_pathnames([fake_fn]) - r.create_filehandlers(loadables) - l1b_base = fake_fn.split(".")[0] - msg = f"Missing navigation donor {l1b_base}" - with pytest.raises(IOError, match=msg): - r.load(["refl_1_38um_nom", "variable2", "variable3"]) - - def test_load_all_old_donor(self): - """Test loading all test datasets with old donor.""" - import xarray as xr - - from satpy.readers import load_reader - r = load_reader(self.reader_configs) - with mock.patch("satpy.readers.clavrx.SDS", xr.DataArray): - loadables = r.select_files_from_pathnames([ - "clavrx_H08_20180806_1800.level2.hdf", - ]) - r.create_filehandlers(loadables) - with mock.patch("satpy.readers.clavrx.glob") as g, mock.patch("satpy.readers.clavrx.netCDF4.Dataset") as d: - g.return_value = ["fake_donor.nc"] - x = np.linspace(-0.1518, 0.1518, 300) - y = np.linspace(0.1518, -0.1518, 10) - proj = mock.Mock( - semi_major_axis=6378.137, - semi_minor_axis=6356.7523142, - perspective_point_height=35791, - longitude_of_projection_origin=140.7, - sweep_angle_axis="y", - ) - d.return_value = fake_donor = mock.MagicMock( - variables={"Projection": proj, "x": x, "y": y}, - ) - fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] - datasets = r.load(["refl_1_38um_nom", "variable2", "variable3"]) - assert len(datasets) == 3 - for v in datasets.values(): - assert "calibration" not in v.attrs - assert v.attrs["units"] in ["1", "%"] - assert isinstance(v.attrs["area"], AreaDefinition) - if v.attrs.get("flag_values"): - assert "_FillValue" in v.attrs - else: - assert "_FillValue" not in v.attrs - if v.attrs["name"] == "refl_1_38um_nom": - assert "valid_range" in v.attrs - assert isinstance(v.attrs["valid_range"], list) - else: - assert "valid_range" not in v.attrs - if "flag_values" in v.attrs: - assert np.issubdtype(v.dtype, np.integer) - assert v.attrs.get("flag_meanings") is not None - - def test_load_all_new_donor(self): - """Test loading all test datasets with new donor.""" - import xarray as xr - - from satpy.readers import load_reader - r = load_reader(self.reader_configs) - with mock.patch("satpy.readers.clavrx.SDS", xr.DataArray): - loadables = r.select_files_from_pathnames([ - "clavrx_H08_20180806_1800.level2.hdf", - ]) - r.create_filehandlers(loadables) - with mock.patch("satpy.readers.clavrx.glob") as g, mock.patch("satpy.readers.clavrx.netCDF4.Dataset") as d: - g.return_value = ["fake_donor.nc"] - x = np.linspace(-0.1518, 0.1518, 300) - y = np.linspace(0.1518, -0.1518, 10) - proj = mock.Mock( - semi_major_axis=6378137, - semi_minor_axis=6356752.3142, - perspective_point_height=35791000, - longitude_of_projection_origin=140.7, - sweep_angle_axis="y", - ) - d.return_value = fake_donor = mock.MagicMock( - variables={"goes_imager_projection": proj, "x": x, "y": y}, - ) - fake_donor.__getitem__.side_effect = lambda key: fake_donor.variables[key] - datasets = r.load(["refl_1_38um_nom", "variable2", "variable3"]) - assert len(datasets) == 3 - for v in datasets.values(): - assert "calibration" not in v.attrs - assert v.attrs["units"] in ["1", "%"] - assert isinstance(v.attrs["area"], AreaDefinition) - assert v.attrs["area"].is_geostationary is True - assert v.attrs["platform_name"] == "himawari8" - assert v.attrs["sensor"] == "ahi" - assert datasets["variable3"].attrs.get("flag_meanings") is not None From e2cf6eb931bf7ffdf8f0230971bb2a73580de286 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Wed, 31 Jan 2024 11:34:13 -0600 Subject: [PATCH 17/23] removing extra lines in creating test content --- satpy/tests/reader_tests/test_clavrx_nc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx_nc.py index 5f3e82f1dc..3cb188d76c 100644 --- a/satpy/tests/reader_tests/test_clavrx_nc.py +++ b/satpy/tests/reader_tests/test_clavrx_nc.py @@ -98,7 +98,6 @@ def fake_test_content(filename, **kwargs): "L1B": "clavrx_H08_20210603_1500_B01_FLDK_R", } ) - variable2 = variable2.where(variable2 % 2 != 0, FILL_VALUE) # category @@ -126,7 +125,6 @@ def fake_test_content(filename, **kwargs): "var_flags": var_flags, "out_of_range_flags": out_of_range_flags, } - ds = xr.Dataset(ds_vars, attrs=attrs) ds = ds.assign_coords({"latitude": latitude, "longitude": longitude}) From f1170f66c702f7169e2f290d8c5ce8c70cdf8d5d Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 2 Feb 2024 10:37:12 -0600 Subject: [PATCH 18/23] Remove extra spaces --- satpy/etc/enhancements/generic.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index 16daa2ff27..63778bd030 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -285,10 +285,10 @@ enhancements: 2, # Probably Cloudy 3, # Cloudy ], - 'colors': [[ 0, 0, 0], # black,-127 = Fill Value - [ 94, 79, 162], # blue, 0 = Clear - [ 73, 228, 242], # cyan, 1 = Probably Clear - [158, 1, 66], # red, 2 = Probably Cloudy + 'colors': [[0, 0, 0], # black,-127 = Fill Value + [94, 79, 162], # blue, 0 = Clear + [73, 228, 242], # cyan, 1 = Probably Clear + [158, 1, 66], # red, 2 = Probably Cloudy [255, 255, 255], # white, 3 = Cloudy ], 'color_scale': 255, From 143973e7239350bc6eb11d01160eb8add895d634 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Mon, 12 Feb 2024 14:08:04 -0600 Subject: [PATCH 19/23] Change area definition and add check for resolution --- satpy/readers/clavrx.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index e23bde44e5..39fda49d3d 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -268,24 +268,14 @@ def _read_axi_fixed_grid(filename: str, sensor: str, l1b_attr) -> geometry.AreaD x, y = l1b["x"], l1b["y"] area_extent, ncols, nlines = _CLAVRxHelper._area_extent(x, y, h) - if sensor == "abi": - area = geometry.AreaDefinition( - "abi_geos", - "ABI L2 file area", - "abi_geos", - proj, - ncols, - nlines, - np.asarray(area_extent)) - else: - area = geometry.AreaDefinition( - "ahi_geos", - "AHI L2 file area", - "ahi_geos", - proj, - ncols, - nlines, - np.asarray(area_extent)) + area = geometry.AreaDefinition( + f"{sensor}_geos", + f"{sensor.upper()} L2 file area", + f"{sensor}_geos", + proj, + ncols, + nlines, + area_extent) return area @@ -515,7 +505,8 @@ def available_datasets(self, configured_datasets=None): # reader knows something about this dataset (file type matches) # add any information that this reader can add. new_info = ds_info.copy() - new_info["resolution"] = self.resolution + if self.resolution is not None: + new_info["resolution"] = self.resolution handled_vars.add(ds_info["name"]) yield True, new_info yield from self._available_file_datasets(handled_vars) From 024a63f957e944828fe7cf558072332a57159398 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 13 Feb 2024 10:41:15 -0600 Subject: [PATCH 20/23] Address: scale_factor/add_offset dtype, area_extent type. Updates logic for the resolution value from the file. --- satpy/readers/clavrx.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/satpy/readers/clavrx.py b/satpy/readers/clavrx.py index 39fda49d3d..c355a1f0ba 100644 --- a/satpy/readers/clavrx.py +++ b/satpy/readers/clavrx.py @@ -17,10 +17,12 @@ # satpy. If not, see . """Interface to CLAVR-X HDF4 products.""" +from __future__ import annotations + import logging import os from glob import glob -from typing import Optional, Union +from typing import Optional import netCDF4 import numpy as np @@ -108,11 +110,11 @@ def _get_rows_per_scan(sensor: str) -> Optional[int]: return None -def _scale_data(data_arr: Union[xr.DataArray, int], scale_factor: float, add_offset: float) -> xr.DataArray: +def _scale_data(data_arr: xr.DataArray | int, scale_factor: float, add_offset: float) -> xr.DataArray: """Scale data, if needed.""" scaling_needed = not (scale_factor == 1.0 and add_offset == 0.0) if scaling_needed: - data_arr = data_arr * scale_factor + add_offset + data_arr = data_arr * np.float32(scale_factor) + np.float32(add_offset) return data_arr @@ -120,16 +122,17 @@ class _CLAVRxHelper: """A base class for the CLAVRx File Handlers.""" @staticmethod - def _get_nadir_resolution(sensor, resolution_from_filename_info): + def _get_nadir_resolution(sensor, filename_info_resolution): """Get nadir resolution.""" for k, v in NADIR_RESOLUTION.items(): if sensor.startswith(k): return v - res = resolution_from_filename_info - if res.endswith("m"): - return int(res[:-1]) - elif res is not None: - return int(res) + if filename_info_resolution is None: + return None + if isinstance(filename_info_resolution, str) and filename_info_resolution.startswith("m"): + return int(filename_info_resolution[:-1]) + else: + return int(filename_info_resolution) @staticmethod def _remove_attributes(attrs: dict) -> dict: From cef6e5a88b67822796952015e49b24bdbd857269 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 13 Feb 2024 11:53:56 -0600 Subject: [PATCH 21/23] Move clavrx tests --- satpy/tests/reader_tests/{ => test_clavrx}/test_clavrx_geohdf.py | 0 satpy/tests/reader_tests/{ => test_clavrx}/test_clavrx_nc.py | 0 .../tests/reader_tests/{ => test_clavrx}/test_clavrx_polarhdf.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename satpy/tests/reader_tests/{ => test_clavrx}/test_clavrx_geohdf.py (100%) rename satpy/tests/reader_tests/{ => test_clavrx}/test_clavrx_nc.py (100%) rename satpy/tests/reader_tests/{ => test_clavrx}/test_clavrx_polarhdf.py (100%) diff --git a/satpy/tests/reader_tests/test_clavrx_geohdf.py b/satpy/tests/reader_tests/test_clavrx/test_clavrx_geohdf.py similarity index 100% rename from satpy/tests/reader_tests/test_clavrx_geohdf.py rename to satpy/tests/reader_tests/test_clavrx/test_clavrx_geohdf.py diff --git a/satpy/tests/reader_tests/test_clavrx_nc.py b/satpy/tests/reader_tests/test_clavrx/test_clavrx_nc.py similarity index 100% rename from satpy/tests/reader_tests/test_clavrx_nc.py rename to satpy/tests/reader_tests/test_clavrx/test_clavrx_nc.py diff --git a/satpy/tests/reader_tests/test_clavrx_polarhdf.py b/satpy/tests/reader_tests/test_clavrx/test_clavrx_polarhdf.py similarity index 100% rename from satpy/tests/reader_tests/test_clavrx_polarhdf.py rename to satpy/tests/reader_tests/test_clavrx/test_clavrx_polarhdf.py From 8d45299e7df011afa1b19f0683cc3fa813337648 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 16 Feb 2024 09:41:05 -0600 Subject: [PATCH 22/23] Add init to clavrx tests directory --- .../tests/reader_tests/test_clavrx/__init__.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 satpy/tests/reader_tests/test_clavrx/__init__.py diff --git a/satpy/tests/reader_tests/test_clavrx/__init__.py b/satpy/tests/reader_tests/test_clavrx/__init__.py new file mode 100644 index 0000000000..6f62e3a26b --- /dev/null +++ b/satpy/tests/reader_tests/test_clavrx/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2017-2018 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""The clavrx reader tests package.""" From f6d1f1ff2c61e47b78e5b47613ef6e874909008f Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 16 Feb 2024 09:57:31 -0600 Subject: [PATCH 23/23] Fix apostrophes being replaced with double quotes --- .../tests/reader_tests/test_clavrx/test_clavrx_polarhdf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/satpy/tests/reader_tests/test_clavrx/test_clavrx_polarhdf.py b/satpy/tests/reader_tests/test_clavrx/test_clavrx_polarhdf.py index 6b69d8a923..f8ae93c38b 100644 --- a/satpy/tests/reader_tests/test_clavrx/test_clavrx_polarhdf.py +++ b/satpy/tests/reader_tests/test_clavrx/test_clavrx_polarhdf.py @@ -186,7 +186,7 @@ def test_available_datasets(self): assert new_ds_infos[1][0] assert new_ds_infos[1][1]["resolution"] == 742 - # we have this, but don"t want to change the resolution + # we have this, but don't want to change the resolution # because a previous handler said it has it assert new_ds_infos[2][0] assert new_ds_infos[2][1]["resolution"] == 1 @@ -201,11 +201,11 @@ def test_available_datasets(self): assert new_ds_infos[4][0] assert new_ds_infos[4][1]["resolution"] == 742 - # we don"t have this variable, don"t change it + # we don"t have this variable, don't change it assert not new_ds_infos[5][0] assert new_ds_infos[5][1].get("resolution") is None - # we have this, but it isn"t supposed to come from our file type + # we have this, but it isn't supposed to come from our file type assert new_ds_infos[6][0] is None assert new_ds_infos[6][1].get("resolution") is None