From fa7942d78f85a6680feca4b8202bed708486df06 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 6 May 2022 11:00:56 +0200 Subject: [PATCH 001/130] to_hvplot function New function to plot Scene datasets as Hvplot Overlay --- AUTHORS.md | 1 + satpy/scene.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index dd2b24750d..e2aa4be396 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -47,6 +47,7 @@ The following people have made contributions to this project: - [Andrea Meraner (ameraner)](https://github.com/ameraner) - [Aronne Merrelli (aronnem)](https://github.com/aronnem) - [Lucas Meyer (LTMeyer)](https://github.com/LTMeyer) +- [Luca Merucci (lmeru)](https://github.com/lmeru) - [Ondrej Nedelcev (nedelceo)](https://github.com/nedelceo) - [Oana Nicola](https://github.com/) - [Esben S. Nielsen (storpipfugl)](https://github.com/storpipfugl) diff --git a/satpy/scene.py b/satpy/scene.py index 261d84ea81..6931c5bc4d 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1021,6 +1021,71 @@ def to_geoviews(self, gvtype=None, datasets=None, kdims=None, vdims=None, dynami gview = gvds.to(gvtype, kdims=["x", "y"], vdims=vdims, dynamic=dynamic) return gview + + def to_hvplot(self,datasets=None, *args,**kwargs): + """ + Convert satpy Scene to Hvplot. + Args: + datasets (list): Limit included products to these datasets. + kwargs: hvplot options list. + + Returns: hvplot object that contains within it the plots of datasets list. + As default it contains all Scene datasets plots and a plot title is shown. + + Example usage: + scene_list = ['ash','IR_108'] + plot = scn.to_hvplot(datasets=scene_list) + + plot.ash+plot.IR_108 + """ + + import hvplot.xarray + from holoviews import Overlay + from satpy import composites + from cartopy import crs + + def _get_crs(xarray_ds): + return xarray_ds.area.to_cartopy_crs() + + def _get_timestamp(xarray_ds): + time = xarray_ds.attrs['start_time'] + return time.strftime('%Y %m %d -- %H:%M UTC') + + def _get_units(xarray_ds,variable): + return xarray_ds[variable].attrs['units'] + + def _plot_rgb(xarray_ds, variable,**defaults): + img = composites.enhance2dataset(xarray_ds[variable]) + return img.hvplot.rgb(bands='bands',title=title, + clabel='',**defaults) + + def _plot_quadmesh(xarray_ds,variable,**defaults): + return xarray_ds[variable].hvplot.quadmesh( + clabel=f'[{_get_units(xarray_ds,variable)}]', + title=title,**defaults) + + plot = Overlay() + xarray_ds = self.to_xarray_dataset(datasets) + ccrs = _get_crs(xarray_ds) + + if datasets is None: datasets = list(xarray_ds.keys()) + + defaults = dict(x='x',y='y',data_aspect=1,project=True,geo=True, + crs=ccrs,projection=ccrs,rasterize=True, + coastline='110m',cmap='Plasma',responsive=True, + dynamic=False,framewise=True,colorbar=False, + global_extent=False,xlabel='Longitude',ylabel='Latitude') + + defaults.update(kwargs) + + for element in datasets: + title = f'{element} @ {_get_timestamp(xarray_ds)}' + if xarray_ds[element].shape[0] == 3: + plot[element] =_plot_rgb(xarray_ds,element,**defaults) + else: + plot[element]=_plot_quadmesh(xarray_ds,element,**defaults) + + return plot def to_xarray_dataset(self, datasets=None): """Merge all xr.DataArrays of a scene to a xr.DataSet. From 88d40023dd250bf4317ddee4d618a9ecf4fdfb66 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 6 May 2022 11:14:34 +0200 Subject: [PATCH 002/130] Add to_hvplot function to_hvplot function plot the Scene datasets as Hvplot Overlay. Added Luca Merucci in authors.md (we create this function together ) --- AUTHORS.md | 1 + satpy/scene.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index dd2b24750d..e2aa4be396 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -47,6 +47,7 @@ The following people have made contributions to this project: - [Andrea Meraner (ameraner)](https://github.com/ameraner) - [Aronne Merrelli (aronnem)](https://github.com/aronnem) - [Lucas Meyer (LTMeyer)](https://github.com/LTMeyer) +- [Luca Merucci (lmeru)](https://github.com/lmeru) - [Ondrej Nedelcev (nedelceo)](https://github.com/nedelceo) - [Oana Nicola](https://github.com/) - [Esben S. Nielsen (storpipfugl)](https://github.com/storpipfugl) diff --git a/satpy/scene.py b/satpy/scene.py index 261d84ea81..6931c5bc4d 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1021,6 +1021,71 @@ def to_geoviews(self, gvtype=None, datasets=None, kdims=None, vdims=None, dynami gview = gvds.to(gvtype, kdims=["x", "y"], vdims=vdims, dynamic=dynamic) return gview + + def to_hvplot(self,datasets=None, *args,**kwargs): + """ + Convert satpy Scene to Hvplot. + Args: + datasets (list): Limit included products to these datasets. + kwargs: hvplot options list. + + Returns: hvplot object that contains within it the plots of datasets list. + As default it contains all Scene datasets plots and a plot title is shown. + + Example usage: + scene_list = ['ash','IR_108'] + plot = scn.to_hvplot(datasets=scene_list) + + plot.ash+plot.IR_108 + """ + + import hvplot.xarray + from holoviews import Overlay + from satpy import composites + from cartopy import crs + + def _get_crs(xarray_ds): + return xarray_ds.area.to_cartopy_crs() + + def _get_timestamp(xarray_ds): + time = xarray_ds.attrs['start_time'] + return time.strftime('%Y %m %d -- %H:%M UTC') + + def _get_units(xarray_ds,variable): + return xarray_ds[variable].attrs['units'] + + def _plot_rgb(xarray_ds, variable,**defaults): + img = composites.enhance2dataset(xarray_ds[variable]) + return img.hvplot.rgb(bands='bands',title=title, + clabel='',**defaults) + + def _plot_quadmesh(xarray_ds,variable,**defaults): + return xarray_ds[variable].hvplot.quadmesh( + clabel=f'[{_get_units(xarray_ds,variable)}]', + title=title,**defaults) + + plot = Overlay() + xarray_ds = self.to_xarray_dataset(datasets) + ccrs = _get_crs(xarray_ds) + + if datasets is None: datasets = list(xarray_ds.keys()) + + defaults = dict(x='x',y='y',data_aspect=1,project=True,geo=True, + crs=ccrs,projection=ccrs,rasterize=True, + coastline='110m',cmap='Plasma',responsive=True, + dynamic=False,framewise=True,colorbar=False, + global_extent=False,xlabel='Longitude',ylabel='Latitude') + + defaults.update(kwargs) + + for element in datasets: + title = f'{element} @ {_get_timestamp(xarray_ds)}' + if xarray_ds[element].shape[0] == 3: + plot[element] =_plot_rgb(xarray_ds,element,**defaults) + else: + plot[element]=_plot_quadmesh(xarray_ds,element,**defaults) + + return plot def to_xarray_dataset(self, datasets=None): """Merge all xr.DataArrays of a scene to a xr.DataSet. From 2fa9b87ef437e63c6c2e37ddec00f7aaf91b2dd5 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 6 May 2022 12:42:35 +0200 Subject: [PATCH 003/130] trying to follow and correct stickler-ci messages --- satpy/scene.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index 6931c5bc4d..83361588dc 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1022,9 +1022,8 @@ def to_geoviews(self, gvtype=None, datasets=None, kdims=None, vdims=None, dynami return gview - def to_hvplot(self,datasets=None, *args,**kwargs): - """ - Convert satpy Scene to Hvplot. + def to_hvplot(self,datasets=None,*args,**kwargs): + """Convert satpy Scene to Hvplot. Args: datasets (list): Limit included products to these datasets. kwargs: hvplot options list. @@ -1037,8 +1036,8 @@ def to_hvplot(self,datasets=None, *args,**kwargs): plot = scn.to_hvplot(datasets=scene_list) plot.ash+plot.IR_108 + """ - import hvplot.xarray from holoviews import Overlay from satpy import composites @@ -1054,7 +1053,7 @@ def _get_timestamp(xarray_ds): def _get_units(xarray_ds,variable): return xarray_ds[variable].attrs['units'] - def _plot_rgb(xarray_ds, variable,**defaults): + def _plot_rgb(xarray_ds,variable,**defaults): img = composites.enhance2dataset(xarray_ds[variable]) return img.hvplot.rgb(bands='bands',title=title, clabel='',**defaults) From fcfd481516b6ebfbe31f80f8395ecb9b67aee71a Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 6 May 2022 13:24:46 +0200 Subject: [PATCH 004/130] correction of whitespaces --- satpy/scene.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index 83361588dc..61d85e3599 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1021,7 +1021,7 @@ def to_geoviews(self, gvtype=None, datasets=None, kdims=None, vdims=None, dynami gview = gvds.to(gvtype, kdims=["x", "y"], vdims=vdims, dynamic=dynamic) return gview - + def to_hvplot(self,datasets=None,*args,**kwargs): """Convert satpy Scene to Hvplot. Args: @@ -1030,29 +1030,29 @@ def to_hvplot(self,datasets=None,*args,**kwargs): Returns: hvplot object that contains within it the plots of datasets list. As default it contains all Scene datasets plots and a plot title is shown. - + Example usage: scene_list = ['ash','IR_108'] plot = scn.to_hvplot(datasets=scene_list) - + plot.ash+plot.IR_108 """ import hvplot.xarray from holoviews import Overlay from satpy import composites - from cartopy import crs - + from cartopy import crs + def _get_crs(xarray_ds): return xarray_ds.area.to_cartopy_crs() def _get_timestamp(xarray_ds): time = xarray_ds.attrs['start_time'] return time.strftime('%Y %m %d -- %H:%M UTC') - + def _get_units(xarray_ds,variable): return xarray_ds[variable].attrs['units'] - + def _plot_rgb(xarray_ds,variable,**defaults): img = composites.enhance2dataset(xarray_ds[variable]) return img.hvplot.rgb(bands='bands',title=title, @@ -1062,11 +1062,11 @@ def _plot_quadmesh(xarray_ds,variable,**defaults): return xarray_ds[variable].hvplot.quadmesh( clabel=f'[{_get_units(xarray_ds,variable)}]', title=title,**defaults) - + plot = Overlay() xarray_ds = self.to_xarray_dataset(datasets) ccrs = _get_crs(xarray_ds) - + if datasets is None: datasets = list(xarray_ds.keys()) defaults = dict(x='x',y='y',data_aspect=1,project=True,geo=True, @@ -1074,18 +1074,18 @@ def _plot_quadmesh(xarray_ds,variable,**defaults): coastline='110m',cmap='Plasma',responsive=True, dynamic=False,framewise=True,colorbar=False, global_extent=False,xlabel='Longitude',ylabel='Latitude') - + defaults.update(kwargs) - + for element in datasets: title = f'{element} @ {_get_timestamp(xarray_ds)}' if xarray_ds[element].shape[0] == 3: plot[element] =_plot_rgb(xarray_ds,element,**defaults) else: plot[element]=_plot_quadmesh(xarray_ds,element,**defaults) - - return plot + return plot + def to_xarray_dataset(self, datasets=None): """Merge all xr.DataArrays of a scene to a xr.DataSet. From dfd93ec74f6f0e1658400f78e46fec2ec03ebedc Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 6 May 2022 13:51:41 +0200 Subject: [PATCH 005/130] correction whitespaces --- satpy/scene.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index 61d85e3599..097c19e521 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1022,7 +1022,7 @@ def to_geoviews(self, gvtype=None, datasets=None, kdims=None, vdims=None, dynami return gview - def to_hvplot(self,datasets=None,*args,**kwargs): + def to_hvplot(self, datasets=None, *args, **kwargs): """Convert satpy Scene to Hvplot. Args: datasets (list): Limit included products to these datasets. @@ -1050,18 +1050,18 @@ def _get_timestamp(xarray_ds): time = xarray_ds.attrs['start_time'] return time.strftime('%Y %m %d -- %H:%M UTC') - def _get_units(xarray_ds,variable): + def _get_units(xarray_ds, variable): return xarray_ds[variable].attrs['units'] - def _plot_rgb(xarray_ds,variable,**defaults): + def _plot_rgb(xarray_ds, variable, **defaults): img = composites.enhance2dataset(xarray_ds[variable]) - return img.hvplot.rgb(bands='bands',title=title, - clabel='',**defaults) + return img.hvplot.rgb(bands='bands', title=title, + clabel='', **defaults) - def _plot_quadmesh(xarray_ds,variable,**defaults): + def _plot_quadmesh(xarray_ds, variable, **defaults): return xarray_ds[variable].hvplot.quadmesh( - clabel=f'[{_get_units(xarray_ds,variable)}]', - title=title,**defaults) + clabel=f'[{_get_units(xarray_ds,variable)}]', title=title, + **defaults) plot = Overlay() xarray_ds = self.to_xarray_dataset(datasets) @@ -1069,20 +1069,19 @@ def _plot_quadmesh(xarray_ds,variable,**defaults): if datasets is None: datasets = list(xarray_ds.keys()) - defaults = dict(x='x',y='y',data_aspect=1,project=True,geo=True, - crs=ccrs,projection=ccrs,rasterize=True, - coastline='110m',cmap='Plasma',responsive=True, - dynamic=False,framewise=True,colorbar=False, - global_extent=False,xlabel='Longitude',ylabel='Latitude') + defaults = dict(x='x', y='y', data_aspect=1, project=True, geo=True, + crs=ccrs, projection=ccrs, rasterize=True, coastline='110m', + cmap='Plasma', responsive=True, dynamic=False, framewise=True, + colorbar=False, global_extent=False, xlabel='Longitude', ylabel='Latitude') defaults.update(kwargs) for element in datasets: title = f'{element} @ {_get_timestamp(xarray_ds)}' if xarray_ds[element].shape[0] == 3: - plot[element] =_plot_rgb(xarray_ds,element,**defaults) + plot[element] = _plot_rgb(xarray_ds, element, **defaults) else: - plot[element]=_plot_quadmesh(xarray_ds,element,**defaults) + plot[element] = _plot_quadmesh(xarray_ds, element, **defaults) return plot From c0022f3da8f24ca9b410c4d0b3159ef4e1d3e929 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 6 May 2022 14:05:17 +0200 Subject: [PATCH 006/130] correction whitespaces --- satpy/scene.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index 097c19e521..2ea3c041c3 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1069,10 +1069,11 @@ def _plot_quadmesh(xarray_ds, variable, **defaults): if datasets is None: datasets = list(xarray_ds.keys()) - defaults = dict(x='x', y='y', data_aspect=1, project=True, geo=True, - crs=ccrs, projection=ccrs, rasterize=True, coastline='110m', - cmap='Plasma', responsive=True, dynamic=False, framewise=True, - colorbar=False, global_extent=False, xlabel='Longitude', ylabel='Latitude') + defaults = dict(x='x', y='y', data_aspect=1, project=True, geo=True, + crs=ccrs, projection=ccrs, rasterize=True, coastline='110m', + cmap='Plasma', responsive=True, dynamic=False, framewise=True, + colorbar=False, global_extent=False, xlabel='Longitude', + ylabel='Latitude') defaults.update(kwargs) From 249c4209c61bdf81f81ebfc81694e0f16fa25e11 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 6 May 2022 19:46:29 +0200 Subject: [PATCH 007/130] function correction for pull request correction whitespaces, import libraries at beginning --- AUTHORS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS.md b/AUTHORS.md index e2aa4be396..85adb23559 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -46,8 +46,8 @@ The following people have made contributions to this project: - [Lu Liu (yukaribbba)](https://github.com/yukaribbba) - [Andrea Meraner (ameraner)](https://github.com/ameraner) - [Aronne Merrelli (aronnem)](https://github.com/aronnem) -- [Lucas Meyer (LTMeyer)](https://github.com/LTMeyer) - [Luca Merucci (lmeru)](https://github.com/lmeru) +- [Lucas Meyer (LTMeyer)](https://github.com/LTMeyer) - [Ondrej Nedelcev (nedelceo)](https://github.com/nedelceo) - [Oana Nicola](https://github.com/) - [Esben S. Nielsen (storpipfugl)](https://github.com/storpipfugl) From 2f2e8a84eb05b07a8a7464e1766873efb90052fc Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 6 May 2022 21:10:21 +0200 Subject: [PATCH 008/130] Add to_hvplot functon --- satpy/scene.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index 2ea3c041c3..ecaa58bfed 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -23,12 +23,14 @@ import warnings from typing import Callable +import hvplot.xarray # noqa import numpy as np import xarray as xr +from holoviews import Overlay from pyresample.geometry import AreaDefinition, BaseDefinition, SwathDefinition from xarray import DataArray -from satpy.composites import IncompatibleAreas +from satpy.composites import IncompatibleAreas, enhance2dataset from satpy.composites.config_loader import load_compositor_configs_for_sensors from satpy.dataset import DataID, DataQuery, DatasetDict, combine_metadata, dataset_walker, replace_anc from satpy.dependency_tree import DependencyTree @@ -1024,11 +1026,12 @@ def to_geoviews(self, gvtype=None, datasets=None, kdims=None, vdims=None, dynami def to_hvplot(self, datasets=None, *args, **kwargs): """Convert satpy Scene to Hvplot. - Args: + + Args: datasets (list): Limit included products to these datasets. kwargs: hvplot options list. - Returns: hvplot object that contains within it the plots of datasets list. + Returns: hvplot object that contains within it the plots of datasets list. As default it contains all Scene datasets plots and a plot title is shown. Example usage: @@ -1038,11 +1041,6 @@ def to_hvplot(self, datasets=None, *args, **kwargs): plot.ash+plot.IR_108 """ - import hvplot.xarray - from holoviews import Overlay - from satpy import composites - from cartopy import crs - def _get_crs(xarray_ds): return xarray_ds.area.to_cartopy_crs() @@ -1054,22 +1052,23 @@ def _get_units(xarray_ds, variable): return xarray_ds[variable].attrs['units'] def _plot_rgb(xarray_ds, variable, **defaults): - img = composites.enhance2dataset(xarray_ds[variable]) + img = enhance2dataset(xarray_ds[variable]) return img.hvplot.rgb(bands='bands', title=title, clabel='', **defaults) def _plot_quadmesh(xarray_ds, variable, **defaults): return xarray_ds[variable].hvplot.quadmesh( - clabel=f'[{_get_units(xarray_ds,variable)}]', title=title, + clabel=f'[{_get_units(xarray_ds,variable)}]', title=title, **defaults) - plot = Overlay() + plot = Overlay() xarray_ds = self.to_xarray_dataset(datasets) ccrs = _get_crs(xarray_ds) - if datasets is None: datasets = list(xarray_ds.keys()) + if datasets is None: + datasets = list(xarray_ds.keys()) - defaults = dict(x='x', y='y', data_aspect=1, project=True, geo=True, + defaults = dict(x='x', y='y', data_aspect=1, project=True, geo=True, crs=ccrs, projection=ccrs, rasterize=True, coastline='110m', cmap='Plasma', responsive=True, dynamic=False, framewise=True, colorbar=False, global_extent=False, xlabel='Longitude', @@ -1085,7 +1084,7 @@ def _plot_quadmesh(xarray_ds, variable, **defaults): plot[element] = _plot_quadmesh(xarray_ds, element, **defaults) return plot - + def to_xarray_dataset(self, datasets=None): """Merge all xr.DataArrays of a scene to a xr.DataSet. From ac8a36167736df51b5f3659be2270520848dd97d Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 6 May 2022 21:26:39 +0200 Subject: [PATCH 009/130] add hvplot in extras require --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0f10d9d15d..2ca2def303 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,7 @@ 'doc': ['sphinx', 'sphinx_rtd_theme', 'sphinxcontrib-apidoc'], # Other 'geoviews': ['geoviews'], + 'hvplot': ['hvplot'], 'overlays': ['pycoast', 'pydecorate'], 'tests': test_requires, } From d2c80fb9876f9aa82527a17a33ed324b2c8d677f Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 6 May 2022 23:24:07 +0200 Subject: [PATCH 010/130] add hvplot in test require --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2ca2def303..099a3f63b7 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,9 @@ test_requires = ['behave', 'h5py', 'netCDF4', 'pyhdf', 'imageio', 'pylibtiff', 'rasterio', 'geoviews', 'trollimage', 'fsspec', 'bottleneck', - 'rioxarray', 'pytest', 'pytest-lazy-fixture', 'defusedxml'] + 'rioxarray', 'pytest', 'pytest-lazy-fixture', 'defusedxml', + 'hvplot'] + extras_require = { # Readers: From 09b8f6ee3252ca52e9ba8fd14ecf69cf7f4b49f9 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Sat, 7 May 2022 19:47:01 +0200 Subject: [PATCH 011/130] Answer to #issuecomment-1120099909 --- continuous_integration/environment.yaml | 1 + satpy/scene.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index d131bad1e0..84c5b4be68 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -35,6 +35,7 @@ dependencies: - mock - libtiff - geoviews + - hvplot - zarr - python-eccodes # 2.19.1 seems to cause library linking issues diff --git a/satpy/scene.py b/satpy/scene.py index ecaa58bfed..c013957c6e 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -23,7 +23,6 @@ import warnings from typing import Callable -import hvplot.xarray # noqa import numpy as np import xarray as xr from holoviews import Overlay @@ -39,6 +38,12 @@ from satpy.resample import get_area_def, prepare_resampler, resample_dataset from satpy.writers import load_writer +try: + import hvplot.xarray # noqa +except ImportError: + hvplot.xarray = None + + LOG = logging.getLogger(__name__) @@ -1061,6 +1066,9 @@ def _plot_quadmesh(xarray_ds, variable, **defaults): clabel=f'[{_get_units(xarray_ds,variable)}]', title=title, **defaults) + if hvplot.xarray is None: + raise ImportError("'hvplot' must be installed to use this feature") + plot = Overlay() xarray_ds = self.to_xarray_dataset(datasets) ccrs = _get_crs(xarray_ds) From d0299dd98254e0a37406e8dee7c3fec6d041c227 Mon Sep 17 00:00:00 2001 From: bornagain Date: Sat, 7 May 2022 21:35:43 +0200 Subject: [PATCH 012/130] Update satpy/scene.py Co-authored-by: Panu Lahtinen --- satpy/scene.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/scene.py b/satpy/scene.py index c013957c6e..4a73f99a14 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1034,7 +1034,7 @@ def to_hvplot(self, datasets=None, *args, **kwargs): Args: datasets (list): Limit included products to these datasets. - kwargs: hvplot options list. + kwargs: hvplot options dictionary. Returns: hvplot object that contains within it the plots of datasets list. As default it contains all Scene datasets plots and a plot title is shown. From aed6a9173fd8bf5dd33e7c8a5ca1ee3755e577a5 Mon Sep 17 00:00:00 2001 From: bornagain Date: Sat, 7 May 2022 21:35:58 +0200 Subject: [PATCH 013/130] Update satpy/scene.py Co-authored-by: Panu Lahtinen --- satpy/scene.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/satpy/scene.py b/satpy/scene.py index 4a73f99a14..1468487da6 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1039,7 +1039,8 @@ def to_hvplot(self, datasets=None, *args, **kwargs): Returns: hvplot object that contains within it the plots of datasets list. As default it contains all Scene datasets plots and a plot title is shown. - Example usage: + Example usage:: + scene_list = ['ash','IR_108'] plot = scn.to_hvplot(datasets=scene_list) From 4687ba437dc029514afe518b96caf7b0bcd83709 Mon Sep 17 00:00:00 2001 From: bornagain Date: Sat, 7 May 2022 21:37:07 +0200 Subject: [PATCH 014/130] Update satpy/scene.py Co-authored-by: Panu Lahtinen --- satpy/scene.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index 1468487da6..7501ee54b7 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -39,9 +39,9 @@ from satpy.writers import load_writer try: - import hvplot.xarray # noqa + import hvplot.xarray as hvplot_xarray # noqa except ImportError: - hvplot.xarray = None + hvplot_xarray = None LOG = logging.getLogger(__name__) From 1aff451cd1481077090bc6da047b96693ad4b588 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Sat, 7 May 2022 21:41:55 +0200 Subject: [PATCH 015/130] Update --- satpy/scene.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/scene.py b/satpy/scene.py index 7501ee54b7..1fcd7fc057 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1067,7 +1067,7 @@ def _plot_quadmesh(xarray_ds, variable, **defaults): clabel=f'[{_get_units(xarray_ds,variable)}]', title=title, **defaults) - if hvplot.xarray is None: + if hvplot_xarray is None: raise ImportError("'hvplot' must be installed to use this feature") plot = Overlay() From 546bb5f938d44ca1231227cd343bd13507c8885b Mon Sep 17 00:00:00 2001 From: bornagain Date: Tue, 2 Aug 2022 08:09:48 +0200 Subject: [PATCH 016/130] Update setup.py Co-authored-by: Panu Lahtinen --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 099a3f63b7..af6819d6e5 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ 'doc': ['sphinx', 'sphinx_rtd_theme', 'sphinxcontrib-apidoc'], # Other 'geoviews': ['geoviews'], - 'hvplot': ['hvplot'], + 'hvplot': ['hvplot', 'geoviews', 'cartopy'], 'overlays': ['pycoast', 'pydecorate'], 'tests': test_requires, } From 590d3742d1a083b8ab87a4a27936ac22c929949d Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 24 Feb 2023 13:59:19 +0100 Subject: [PATCH 017/130] add holoviews to continuous_integration --- continuous_integration/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index 57728e2c73..67c007a798 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -35,12 +35,12 @@ dependencies: - mock - libtiff - geoviews + - holoviews - hvplot - zarr - python-eccodes # 2.19.1 seems to cause library linking issues - eccodes>=2.20 - - geoviews - pytest - pytest-cov - pytest-lazy-fixture From 24273d57e8c1d05b62c5104e53f0383e1e29ad40 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Mon, 20 Mar 2023 13:46:53 +0100 Subject: [PATCH 018/130] Add compositor for high-level clouds following GeoColor implementation. --- satpy/composites/__init__.py | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 0948d543ab..248aa3b8e6 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1018,6 +1018,74 @@ def __call__(self, projectables, **kwargs): return res +class HighCloudCompositor(CloudCompositor): + """Detect high clouds based on latitude-dependent thresholding and use it as a mask for compositing. + + This compositor aims at identifying high clouds and assigning them a transparency based on the brightness + temperature (cloud opacity). In contrast to the `CloudCompositor`, the brightness temperature threshold at + the lower end, used to identify opaque clouds, is made a function of the latitude in order to have tropopause + level clouds appear as opaque at both high and low latitudes. This follows the Geocolor implementation of + high clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`). + + The idea is to define a tuple of two brightness temperature thresholds in transisiton_min and two corresponding + latitude thresholds in latitude_min. + + + TODO improve docstring: + The modified and latitude-dependent transition_min, sent to `CloudCopositor`, + will then be computed such that transition_min[0] is used if abs(latitude) < latitude_min[0]. + + if abs(latitude) < latitude_min(0): + tr_min_lat = transition_min[0] + elif abs(latitude) > latitude_min(1): + tr_min_lat = transition_min[1] + else: + tr_min_lat = linear intterpolation of + + tr_min_lat = transition_min[0] where abs(latitude) < latitude_min(0) + tr_min_lat = transition_min[1] where abs(latitude) > latitude_min(0) + tr_min_lat = linear interpolation between transition_min[0] and transition_min[1] where abs(latitude). + + """ + + def __init__(self, name, transition_min=(200., 220.), transition_max=280, latitude_min=(30., 60.), + transition_gamma=1.0, **kwargs): + """Collect custom configuration values. + + Args: + transition_min (tuple): Brightness temperature values used to identify opaque white + clouds at different latitudes + transition_max (float): Brightness temperatures above this value are not considered to + be high clouds -> transparent + latitude_min (tuple): Latitude values defining the intervals for computing latitude-dependent + transition_min values. + transition_gamma (float): Gamma correction to apply at the end + + """ + self.latitude_min = latitude_min + super().__init__(name, transition_min=transition_min, transition_max=transition_max, + transition_gamma=transition_gamma, **kwargs) + + def __call__(self, projectables, **kwargs): + """Generate the composite.""" + data = projectables[0] + _, lats = data.attrs["area"].get_lonlats() + lats = np.abs(lats) + + slope = (self.transition_min[1] - self.transition_min[0]) / (self.latitude_min[1] - self.latitude_min[0]) + offset = self.transition_min[0] - slope * self.latitude_min[0] + + tr_min_lat = xr.DataArray(name='tr_min_lat', coords=data.coords, dims=data.dims) + tr_min_lat = tr_min_lat.where(lats >= self.latitude_min[0], self.transition_min[0]) + tr_min_lat = tr_min_lat.where(lats <= self.latitude_min[1], self.transition_min[1]) + tr_min_lat = tr_min_lat.where((lats < self.latitude_min[0]) | (lats > self.latitude_min[1]), + slope * lats + offset) + + self.transition_min = tr_min_lat + + return super().__call__(projectables, **kwargs) + + class RatioSharpenedRGB(GenericCompositor): """Sharpen RGB bands with ratio of a high resolution band to a lower resolution version. From d01760c0da297c3bd8dfc984a4edb7d19b3f8960 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Mon, 20 Mar 2023 13:47:53 +0100 Subject: [PATCH 019/130] Add dedicated enhancement recipe for GeoColor high cloud composites. --- satpy/etc/enhancements/generic.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index ce1ce1bb94..b575d776df 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -838,6 +838,20 @@ enhancements: kwargs: weight: 1.0 + ir_high_cloud: + standard_name: ir_high_cloud + operations: + - name: inverse + method: !!python/name:satpy.enhancements.invert + args: + - [True, false] + - name: stretch + method: !!python/name:satpy.enhancements.stretch + kwargs: + stretch: linear + - name: 3d + method: !!python/name:satpy.enhancements.three_d_effect + colorized_ir_clouds: standard_name: colorized_ir_clouds operations: From 84ae8071c956006a49422bd5e419b7c012681a0d Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Mon, 20 Mar 2023 13:49:46 +0100 Subject: [PATCH 020/130] Add AHI recipe for GeoColor high-level cloud composite nighttime layer. --- satpy/etc/composites/ahi.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index a2e80a4ac1..0785ad80c2 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -489,3 +489,10 @@ composites: prerequisites: - night_ir_alpha - _night_background_hires + + # GeoColor + GeoColor_HighClouds: + standard_name: ir_high_cloud + compositor: !!python/name:satpy.composites.HighCloudCompositor + prerequisites: + - name: B13 From fdd084359657c36c21d4c619c004eea8a07e0743 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Wed, 22 Mar 2023 12:00:29 +0100 Subject: [PATCH 021/130] Add first version of low-level cloud composite. --- satpy/composites/__init__.py | 72 ++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 248aa3b8e6..5c16e0c83f 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1086,6 +1086,78 @@ def __call__(self, projectables, **kwargs): return super().__call__(projectables, **kwargs) +# class LowCloudCompositor(CloudCompositor): +class LowCloudCompositor(GenericCompositor): + """Class information. + + TODO: Rewrite docstring + + Detect low clouds based on latitude-dependent thresholding and use it as a mask for compositing. + + This compositor aims at identifying high clouds and assigning them a transparency based on the brightness + temperature (cloud opacity). In contrast to the `CloudCompositor`, the brightness temperature threshold at + the lower end, used to identify opaque clouds, is made a function of the latitude in order to have tropopause + level clouds appear as opaque at both high and low latitudes. This follows the Geocolor implementation of + high clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`). + + The idea is to define a tuple of two brightness temperature thresholds in transisiton_min and two corresponding + latitude thresholds in latitude_min. + + """ + + def __init__(self, name, range_land=(1.0, 4.5), range_water=(0.0, 4.0), transition_gamma=1.0, + color=(140.25, 191.25, 249.9), **kwargs): + """Init info. + + TODO: Rewrite docstring + Collect custom configuration values. + + Args: + transition_min (tuple): Brightness temperature values used to identify opaque white + clouds at different latitudes + transition_max (float): Brightness temperatures above this value are not considered to + be high clouds -> transparent + latitude_min (tuple): Latitude values defining the intervals for computing latitude-dependent + transition_min values. + transition_gamma (float): Gamma correction to apply at the end + + """ + self.range_land = range_land + self.range_water = range_water + self.transition_gamma = transition_gamma + self.color = color + super().__init__(name, **kwargs) + # super().__init__(name, transition_gamma=transition_gamma, **kwargs) + + def __call__(self, projectables, **kwargs): + """Generate the composite.""" + diff_comp = DifferenceCompositor(name='ir_difference') + btd = diff_comp.__call__(projectables) + + # Avoid spurious false alarms caused by noise in the 3.9um channel that can occur for very cold cloud tops + btd = btd.where(projectables[0] >= 230, 0.0) + + # self.transition_min, self.transition_max = self.range_land + # res_land = super().__call__((btd), **kwargs) + + # self.transition_min, self.transition_max = self.range_water + # res_water = super().__call__(btd, **kwargs) + + tr_min = self.range_land[0] + tr_max = self.range_land[1] + + slope = 1 / (tr_max - tr_min) + offset = 0 - slope * tr_min + + alpha = btd.where(btd > tr_min, 0.0) + alpha = alpha.where(btd <= tr_max, 1.0) + alpha = alpha.where((btd <= tr_min) | (btd > tr_max), slope * btd + offset) + + alpha **= self.transition_gamma + res = super().__call__((btd, alpha), low_cloud_color=self.color, **kwargs) + return res + + class RatioSharpenedRGB(GenericCompositor): """Sharpen RGB bands with ratio of a high resolution band to a lower resolution version. From 92607d6f07e353494244b5143d9c585466203dcc Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Wed, 22 Mar 2023 12:02:21 +0100 Subject: [PATCH 022/130] Add support for monochromatic colorization enhancement. --- satpy/enhancements/__init__.py | 43 ++++++++++++++++++++++------- satpy/etc/enhancements/generic.yaml | 14 ++++++++++ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/satpy/enhancements/__init__.py b/satpy/enhancements/__init__.py index 29f2cbdf54..0ae8fd45c0 100644 --- a/satpy/enhancements/__init__.py +++ b/satpy/enhancements/__init__.py @@ -358,16 +358,39 @@ def _merge_colormaps(kwargs, img=None): from trollimage.colormap import Colormap full_cmap = None - palette = kwargs['palettes'] - if isinstance(palette, Colormap): - full_cmap = palette - else: - for itm in palette: - cmap = create_colormap(itm, img) - if full_cmap is None: - full_cmap = cmap - else: - full_cmap = full_cmap + cmap + # TODO + # - Improve check if both palettes and monochromatic are set + # - Improve exception handling for monochromatic cases + # - Resolve RunTimeWarnings + + if 'palettes' in kwargs: + palette = kwargs['palettes'] + if isinstance(palette, Colormap): + full_cmap = palette + else: + for itm in palette: + cmap = create_colormap(itm, img) + if full_cmap is None: + full_cmap = cmap + else: + full_cmap = full_cmap + cmap + + if 'monochromatic' in kwargs: + palette = {} + color = kwargs['monochromatic'].get('color', None) + if color is None: + # TODO: add error + pass + elif isinstance(color, (list, tuple)): + palette['colors'] = [color] + elif isinstance(color, str): + var = img.data.attrs.get(color, None) + if not isinstance(var, (tuple, list)): + # TODO: add error + pass + palette['colors'] = [var] + + full_cmap = create_colormap(palette, img) return full_cmap diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index b575d776df..43b40e2fcd 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -852,6 +852,20 @@ enhancements: - name: 3d method: !!python/name:satpy.enhancements.three_d_effect + ir_low_cloud: + standard_name: ir_low_cloud + operations: + - name: stretch + method: !!python/name:satpy.enhancements.stretch + kwargs: + stretch: linear + - name: colorize + method: !!python/name:satpy.enhancements.colorize + kwargs: + monochromatic: + color: low_cloud_color +# color: [0, 255, 0] + colorized_ir_clouds: standard_name: colorized_ir_clouds operations: From 6ee859b9f0d0b2c6a2e10994072dbc6cdae04523 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Wed, 22 Mar 2023 12:03:13 +0100 Subject: [PATCH 023/130] Add GeoColor low-level cloud composite for AHI. --- satpy/etc/composites/ahi.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 0785ad80c2..964e37b0a3 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -491,8 +491,16 @@ composites: - _night_background_hires # GeoColor - GeoColor_HighClouds: + geo_color_high_clouds: standard_name: ir_high_cloud compositor: !!python/name:satpy.composites.HighCloudCompositor prerequisites: - name: B13 + + geo_color_low_clouds: + standard_name: ir_low_cloud + compositor: !!python/name:satpy.composites.LowCloudCompositor + color: [140.25, 191.25, 249.9] + prerequisites: + - name: B13 + - name: B07 From c8ae060e8db938108e2d8e4de6c0eba24abe0f91 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Fri, 31 Mar 2023 18:16:35 +0200 Subject: [PATCH 024/130] Implement usage of land-sea-mask for the brightness temperature difference used for the Geocolor low-level cloud layer. --- satpy/composites/__init__.py | 47 ++++++++++++++--------------- satpy/etc/composites/ahi.yaml | 10 +++++- satpy/etc/enhancements/generic.yaml | 5 ++- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 5c16e0c83f..de7cf04613 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1086,8 +1086,7 @@ def __call__(self, projectables, **kwargs): return super().__call__(projectables, **kwargs) -# class LowCloudCompositor(CloudCompositor): -class LowCloudCompositor(GenericCompositor): +class LowCloudCompositor(CloudCompositor): """Class information. TODO: Rewrite docstring @@ -1105,8 +1104,10 @@ class LowCloudCompositor(GenericCompositor): """ - def __init__(self, name, range_land=(1.0, 4.5), range_water=(0.0, 4.0), transition_gamma=1.0, - color=(140.25, 191.25, 249.9), **kwargs): + def __init__(self, name, land_sea_mask=None, value_land=1, value_sea=0, + range_land=(1.0, 4.5), + range_sea=(0.0, 4.0), + transition_gamma=1.0, color=(140.25, 191.25, 249.9), **kwargs): """Init info. TODO: Rewrite docstring @@ -1122,39 +1123,35 @@ def __init__(self, name, range_land=(1.0, 4.5), range_water=(0.0, 4.0), transiti transition_gamma (float): Gamma correction to apply at the end """ + self.land_sea_mask = land_sea_mask + self.val_land = value_land + self.val_sea = value_sea self.range_land = range_land - self.range_water = range_water + self.range_sea = range_sea self.transition_gamma = transition_gamma self.color = color - super().__init__(name, **kwargs) - # super().__init__(name, transition_gamma=transition_gamma, **kwargs) + self.transition_min = None + self.transition_max = None + super().__init__(name, transition_gamma=transition_gamma, **kwargs) def __call__(self, projectables, **kwargs): """Generate the composite.""" - diff_comp = DifferenceCompositor(name='ir_difference') - btd = diff_comp.__call__(projectables) + projectables = self.match_data_arrays(projectables) + btd, lsm, win_bt = projectables + lsm = lsm.squeeze(drop=True) + lsm = lsm.round() # Make sure to have whole numbers in case of smearing from resampling # Avoid spurious false alarms caused by noise in the 3.9um channel that can occur for very cold cloud tops - btd = btd.where(projectables[0] >= 230, 0.0) - - # self.transition_min, self.transition_max = self.range_land - # res_land = super().__call__((btd), **kwargs) - - # self.transition_min, self.transition_max = self.range_water - # res_water = super().__call__(btd, **kwargs) + btd = btd.where(win_bt >= 230, 0.0) - tr_min = self.range_land[0] - tr_max = self.range_land[1] + self.transition_min, self.transition_max = self.range_land + res = super().__call__([btd.where(lsm == self.val_land)], low_cloud_color=self.color, **kwargs) - slope = 1 / (tr_max - tr_min) - offset = 0 - slope * tr_min + self.transition_min, self.transition_max = self.range_sea + res_sea = super().__call__([btd.where(lsm == self.val_sea)], low_cloud_color=self.color, **kwargs) - alpha = btd.where(btd > tr_min, 0.0) - alpha = alpha.where(btd <= tr_max, 1.0) - alpha = alpha.where((btd <= tr_min) | (btd > tr_max), slope * btd + offset) + res = res.where(lsm == self.val_land, res_sea) - alpha **= self.transition_gamma - res = super().__call__((btd, alpha), low_cloud_color=self.color, **kwargs) return res diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 964e37b0a3..2730547f56 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -500,7 +500,15 @@ composites: geo_color_low_clouds: standard_name: ir_low_cloud compositor: !!python/name:satpy.composites.LowCloudCompositor + value_sea: 0 + value_land: 254 color: [140.25, 191.25, 249.9] prerequisites: + - compositor: !!python/name:satpy.composites.DifferenceCompositor + prerequisites: + - name: B13 + - name: B07 + - compositor: !!python/name:satpy.composites.StaticImageCompositor + standard_name: land_sea_mask + filename: "/tcenas/scratch/strandgren/GeoColor/land_sea_mask_3km_i_.tif" - name: B13 - - name: B07 diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index 43b40e2fcd..2094327918 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -855,6 +855,10 @@ enhancements: ir_low_cloud: standard_name: ir_low_cloud operations: + - name: inverse + method: !!python/name:satpy.enhancements.invert + args: + - [False, True] - name: stretch method: !!python/name:satpy.enhancements.stretch kwargs: @@ -864,7 +868,6 @@ enhancements: kwargs: monochromatic: color: low_cloud_color -# color: [0, 255, 0] colorized_ir_clouds: standard_name: colorized_ir_clouds From c3636f3b3f60a46a6032bdca82a6a8e08b6a50ea Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Mon, 3 Apr 2023 11:33:49 +0200 Subject: [PATCH 025/130] Add GeoColor composite recipes including rayleigh correction modifier. --- satpy/etc/composites/ahi.yaml | 86 ++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 2730547f56..a2fccee84b 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -14,6 +14,22 @@ modifiers: - solar_azimuth_angle - solar_zenith_angle + geo_color_rayleigh_corrected: + modifier: !!python/name:satpy.modifiers.PSPRayleighReflectance + atmosphere: us-standard + aerosol_type: rayleigh_only + reduce_lim_low: 70 + reduce_lim_high: 105 + reduce_strength: 1.5 + prerequisites: + - name: B03 + modifiers: [sunz_corrected] + optional_prerequisites: + - satellite_azimuth_angle + - satellite_zenith_angle + - solar_azimuth_angle + - solar_zenith_angle + composites: green: deprecation_warning: "'green' is a deprecated composite. Use the equivalent 'hybrid_green' instead." @@ -103,17 +119,6 @@ composites: - wavelength: 0.85 standard_name: toa_reflectance - ndvi_hybrid_green: - compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen - prerequisites: - - name: B02 - modifiers: [sunz_corrected, rayleigh_corrected] - - name: B03 - modifiers: [sunz_corrected, rayleigh_corrected] - - name: B04 - modifiers: [sunz_corrected] - standard_name: toa_bidirectional_reflectance - airmass: # PDF slides: https://www.eumetsat.int/website/home/News/ConferencesandEvents/DAT_2833302.html # Under session 2 by Akihiro Shimizu (JMA) @@ -271,17 +276,6 @@ composites: high_resolution_band: red standard_name: true_color - true_color_ndvi_green: - compositor: !!python/name:satpy.composites.SelfSharpenedRGB - prerequisites: - - name: B03 - modifiers: [sunz_corrected, rayleigh_corrected] - - name: ndvi_hybrid_green - - name: B01 - modifiers: [sunz_corrected, rayleigh_corrected] - high_resolution_band: red - standard_name: true_color - natural_color_nocorr: compositor: !!python/name:satpy.composites.SelfSharpenedRGB prerequisites: @@ -491,6 +485,40 @@ composites: - _night_background_hires # GeoColor + + geo_color: + compositor: !!python/name:satpy.composites.DayNightCompositor + lim_low: 80 + lim_high: 88 + standard_name: true_color_with_night_ir + prerequisites: + - geo_color_true_color + - geo_color_night + + # GeoColor Daytime + geo_color_green: + compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen + prerequisites: + - name: B02 + modifiers: [ sunz_corrected, geo_color_rayleigh_corrected ] + - name: B03 + modifiers: [ sunz_corrected, geo_color_rayleigh_corrected ] + - name: B04 + modifiers: [ sunz_corrected ] + standard_name: toa_bidirectional_reflectance + + geo_color_true_color: + compositor: !!python/name:satpy.composites.SelfSharpenedRGB + prerequisites: + - name: B03 + modifiers: [ sunz_corrected, geo_color_rayleigh_corrected ] + - name: geo_color_green + - name: B01 + modifiers: [ sunz_corrected, geo_color_rayleigh_corrected ] + high_resolution_band: red + standard_name: true_color + + # GeoColor Nighttime geo_color_high_clouds: standard_name: ir_high_cloud compositor: !!python/name:satpy.composites.HighCloudCompositor @@ -512,3 +540,17 @@ composites: standard_name: land_sea_mask filename: "/tcenas/scratch/strandgren/GeoColor/land_sea_mask_3km_i_.tif" - name: B13 + + geo_color_bl: + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: night_ir_with_background + prerequisites: + - geo_color_low_clouds + - _night_background_hires + + geo_color_night: + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: night_ir_with_background + prerequisites: + - geo_color_high_clouds + - geo_color_bl From 258099073b32003402f308b1518f308b70e65334 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Mon, 3 Apr 2023 17:08:32 +0200 Subject: [PATCH 026/130] Add AHI GeoColor composite without background layer. --- satpy/etc/composites/ahi.yaml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index a2fccee84b..98e3a028c7 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -495,6 +495,15 @@ composites: - geo_color_true_color - geo_color_night + geo_color_without_background: + compositor: !!python/name:satpy.composites.DayNightCompositor + lim_low: 80 + lim_high: 88 + standard_name: true_color_with_night_ir + prerequisites: + - geo_color_true_color + - geo_color_night_without_background + # GeoColor Daytime geo_color_green: compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen @@ -518,7 +527,7 @@ composites: high_resolution_band: red standard_name: true_color - # GeoColor Nighttime + # GeoColor Night-time geo_color_high_clouds: standard_name: ir_high_cloud compositor: !!python/name:satpy.composites.HighCloudCompositor @@ -538,10 +547,11 @@ composites: - name: B07 - compositor: !!python/name:satpy.composites.StaticImageCompositor standard_name: land_sea_mask + # TODO Change filename filename: "/tcenas/scratch/strandgren/GeoColor/land_sea_mask_3km_i_.tif" - name: B13 - geo_color_bl: + geo_color_background_with_low_clouds: compositor: !!python/name:satpy.composites.BackgroundCompositor standard_name: night_ir_with_background prerequisites: @@ -553,4 +563,11 @@ composites: standard_name: night_ir_with_background prerequisites: - geo_color_high_clouds - - geo_color_bl + - geo_color_background_with_low_clouds + + geo_color_night_without_background: + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: night_ir_with_background + prerequisites: + - geo_color_low_clouds + - geo_color_high_clouds From 608233fb1fa98850ff070848441750139baec55f Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Mon, 3 Apr 2023 18:56:09 +0200 Subject: [PATCH 027/130] Improve docstrings and exception handling of HighCloudCompositor and LowCloudCompositor. --- satpy/composites/__init__.py | 142 +++++++++++++++++++++-------------- 1 file changed, 85 insertions(+), 57 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index de7cf04613..b20d057ca2 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1023,29 +1023,20 @@ class HighCloudCompositor(CloudCompositor): This compositor aims at identifying high clouds and assigning them a transparency based on the brightness temperature (cloud opacity). In contrast to the `CloudCompositor`, the brightness temperature threshold at - the lower end, used to identify opaque clouds, is made a function of the latitude in order to have tropopause - level clouds appear as opaque at both high and low latitudes. This follows the Geocolor implementation of - high clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`). - - The idea is to define a tuple of two brightness temperature thresholds in transisiton_min and two corresponding - latitude thresholds in latitude_min. - - - TODO improve docstring: - The modified and latitude-dependent transition_min, sent to `CloudCopositor`, - will then be computed such that transition_min[0] is used if abs(latitude) < latitude_min[0]. - - if abs(latitude) < latitude_min(0): - tr_min_lat = transition_min[0] - elif abs(latitude) > latitude_min(1): - tr_min_lat = transition_min[1] - else: - tr_min_lat = linear intterpolation of - - tr_min_lat = transition_min[0] where abs(latitude) < latitude_min(0) - tr_min_lat = transition_min[1] where abs(latitude) > latitude_min(0) - tr_min_lat = linear interpolation between transition_min[0] and transition_min[1] where abs(latitude). - + the lower end, used to identify high opaque clouds, is made a function of the latitude in order to have + tropopause level clouds appear opaque at both high and low latitudes. This follows the Geocolor + implementation of high clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`). + + The two brightness temperature thresholds in `transition_min` are used together with the corresponding + latitude limits in `latitude_min` to compute a modified version of `transition_min` that is later used + when calling `CloudCompositor`. The modified version of `transition_min` will be an array with the same + shape as the input projectable dataset, where the actual values of threshold_min are a function of the + dataset `latitude`: + + - transition_min = transition_min[0] where abs(latitude) < latitude_min(0) + - transition_min = transition_min[1] where abs(latitude) > latitude_min(0) + - transition_min = linear interpolation between transition_min[0] and transition_min[1] as a funtion + of where abs(latitude). """ def __init__(self, name, transition_min=(200., 220.), transition_max=280, latitude_min=(30., 60.), @@ -1054,20 +1045,35 @@ def __init__(self, name, transition_min=(200., 220.), transition_max=280, latitu Args: transition_min (tuple): Brightness temperature values used to identify opaque white - clouds at different latitudes + clouds at different latitudes transition_max (float): Brightness temperatures above this value are not considered to be high clouds -> transparent latitude_min (tuple): Latitude values defining the intervals for computing latitude-dependent transition_min values. - transition_gamma (float): Gamma correction to apply at the end + transition_gamma (float): Gamma correction to apply to the alpha channel within the brightness + temperature range (`transition_min` to `transition_max`). """ + if len(transition_min) != 2: + raise ValueError(f"Expected 2 `transition_min` values, got {len(transition_min)}") + if len(latitude_min) != 2: + raise ValueError(f"Expected 2 `latitude_min` values, got {len(latitude_min)}") + if type(transition_max) in [list, tuple]: + raise ValueError(f"Expected `transition_max` to be of type float, is of type {type(transition_max)}") + self.latitude_min = latitude_min super().__init__(name, transition_min=transition_min, transition_max=transition_max, transition_gamma=transition_gamma, **kwargs) def __call__(self, projectables, **kwargs): - """Generate the composite.""" + """Generate the composite. + + `projectables` is expected to be a list or tuple with a single element: + - index 0: Brightness temperature of a thermal infrared window channel (e.g. 10.5 microns). + """ + if len(projectables) != 1: + raise ValueError(f"Expected 1 dataset, got {len(projectables)}") + data = projectables[0] _, lats = data.attrs["area"].get_lonlats() lats = np.abs(lats) @@ -1087,70 +1093,92 @@ def __call__(self, projectables, **kwargs): class LowCloudCompositor(CloudCompositor): - """Class information. + """Detect low-level clouds based on thresholding and use it as a mask for compositing during night-time. - TODO: Rewrite docstring + This compsitor computes the brightness temperature difference between a window channel (e.g. 10.5 micron) + and the near-infrared channel e.g. (3.8 micron) and uses this brightness temperature difference, `BTD`, to + create a partially transparent mask for compositing. - Detect low clouds based on latitude-dependent thresholding and use it as a mask for compositing. - - This compositor aims at identifying high clouds and assigning them a transparency based on the brightness - temperature (cloud opacity). In contrast to the `CloudCompositor`, the brightness temperature threshold at - the lower end, used to identify opaque clouds, is made a function of the latitude in order to have tropopause - level clouds appear as opaque at both high and low latitudes. This follows the Geocolor implementation of - high clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`). - - The idea is to define a tuple of two brightness temperature thresholds in transisiton_min and two corresponding - latitude thresholds in latitude_min. + Pixels with `BTD` values below a given threshold will be transparent, whereas pixels with `BTD` values + above another threshold will be opaque. The transparency of all other `BTD` values will be a linear + function of the `BTD` value itself. Two sets of thresholds are used, one set for land surface types + (`range_land`) and another one for sea/water surface types (`range_sea`), respectively. Hence, + this compositor requires a land-sea-mask as a prerequisite input. This follows the GeoColor + implementation of night-time low-level clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`). + Please note that the spectral test and thus the output of the compositor (using the expected input data) is + only applicable during night-time. """ - def __init__(self, name, land_sea_mask=None, value_land=1, value_sea=0, + def __init__(self, name, values_land=(1), values_sea=(0), range_land=(1.0, 4.5), range_sea=(0.0, 4.0), transition_gamma=1.0, color=(140.25, 191.25, 249.9), **kwargs): """Init info. - TODO: Rewrite docstring Collect custom configuration values. Args: - transition_min (tuple): Brightness temperature values used to identify opaque white - clouds at different latitudes - transition_max (float): Brightness temperatures above this value are not considered to - be high clouds -> transparent + value_land (list): List of values used to identify land surface pixels in the land-sea-mask. + value_sea (list): List of values used to identify sea/water surface pixels in the land-sea-mask. + range_land (tuple): Threshold values used for masking low-level clouds from the brightness temperature + difference over land surface types. + range_sea (tuple): Threshold values used for masking low-level clouds from the brightness temperature + difference over sea/water. latitude_min (tuple): Latitude values defining the intervals for computing latitude-dependent transition_min values. - transition_gamma (float): Gamma correction to apply at the end - + transition_gamma (float): Gamma correction to apply to the alpha channel within the brightness + temperature difference range. + color (list): RGB definition of color to use for the low-level clouds in the composite (the final + color will be a function of the corresponding trasnparency/alpha channel). """ - self.land_sea_mask = land_sea_mask - self.val_land = value_land - self.val_sea = value_sea + if len(range_land) != 2: + raise ValueError(f"Expected 2 `range_land` values, got {len(range_land)}") + if len(range_sea) != 2: + raise ValueError(f"Expected 2 `range_sea` values, got {len(range_sea)}") + if type(color) not in [list, tuple] or len(color) != 3: + raise ValueError("Expected list/tuple with the red, green and blue color components.") + + self.values_land = values_land if type(values_land) in [list, tuple] else [values_land] + self.values_sea = values_sea if type(values_sea) in [list, tuple] else [values_sea] self.range_land = range_land self.range_sea = range_sea self.transition_gamma = transition_gamma self.color = color - self.transition_min = None - self.transition_max = None + self.transition_min = None # Placeholder for later use in CloudCompositor + self.transition_max = None # Placeholder for later use in CloudCompositor super().__init__(name, transition_gamma=transition_gamma, **kwargs) def __call__(self, projectables, **kwargs): - """Generate the composite.""" + """Generate the composite. + + `projectables` is expected to be a list or tuple with the following three elements: + - index 0: Brightness temperature difference between a window channel (e.g. 10.5 micron) and a + near-infrared channel e.g. (3.8 micron). + - index 1. Brightness temperature of the window channel (used to filter out noise-induced false alarms). + - index 2: Land-Sea-Mask. + """ + if len(projectables) != 3: + raise ValueError(f"Expected 3 datasets, got {len(projectables)}") + projectables = self.match_data_arrays(projectables) - btd, lsm, win_bt = projectables + btd, bt_win, lsm = projectables lsm = lsm.squeeze(drop=True) lsm = lsm.round() # Make sure to have whole numbers in case of smearing from resampling # Avoid spurious false alarms caused by noise in the 3.9um channel that can occur for very cold cloud tops - btd = btd.where(win_bt >= 230, 0.0) + btd = btd.where(bt_win >= 230, 0.0) + # Call CloudCompositor for land surface pixels self.transition_min, self.transition_max = self.range_land - res = super().__call__([btd.where(lsm == self.val_land)], low_cloud_color=self.color, **kwargs) + res = super().__call__([btd.where(lsm.isin(self.values_land))], low_cloud_color=self.color, **kwargs) + # Call CloudCompositor for sea/water surface pixels self.transition_min, self.transition_max = self.range_sea - res_sea = super().__call__([btd.where(lsm == self.val_sea)], low_cloud_color=self.color, **kwargs) + res_sea = super().__call__([btd.where(lsm.isin(self.values_sea))], low_cloud_color=self.color, **kwargs) - res = res.where(lsm == self.val_land, res_sea) + # Compine resutls for land and sea/water surface pixels + res = res.where(lsm.isin(self.values_land), res_sea) return res From 489069551915d6c4744856abf65f9e288619215a Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Mon, 3 Apr 2023 18:57:15 +0200 Subject: [PATCH 028/130] Modify low-level cloud composite recipe to account for modified order of projectables and new land-sea mask data. --- satpy/etc/composites/ahi.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 98e3a028c7..f0543dd7d5 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -537,19 +537,19 @@ composites: geo_color_low_clouds: standard_name: ir_low_cloud compositor: !!python/name:satpy.composites.LowCloudCompositor - value_sea: 0 - value_land: 254 + values_sea: 0 + values_land: 100 color: [140.25, 191.25, 249.9] prerequisites: - compositor: !!python/name:satpy.composites.DifferenceCompositor prerequisites: - name: B13 - name: B07 + - name: B13 - compositor: !!python/name:satpy.composites.StaticImageCompositor standard_name: land_sea_mask # TODO Change filename - filename: "/tcenas/scratch/strandgren/GeoColor/land_sea_mask_3km_i_.tif" - - name: B13 + filename: "/tcenas/scratch/strandgren/GeoColor/gshhs_land_sea_mask_3km_i.tif" geo_color_background_with_low_clouds: compositor: !!python/name:satpy.composites.BackgroundCompositor From 40ded69a74d51790ba581e0cf30d4155162a23a3 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Mon, 3 Apr 2023 19:00:39 +0200 Subject: [PATCH 029/130] Fix syntax of single-element tuples. --- satpy/composites/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index b20d057ca2..995ef7d317 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1110,7 +1110,7 @@ class LowCloudCompositor(CloudCompositor): only applicable during night-time. """ - def __init__(self, name, values_land=(1), values_sea=(0), + def __init__(self, name, values_land=(1,), values_sea=(0,), range_land=(1.0, 4.5), range_sea=(0.0, 4.0), transition_gamma=1.0, color=(140.25, 191.25, 249.9), **kwargs): From bd8494e1931d8f967fdb170be218c1a49860b545 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Wed, 5 Apr 2023 13:01:11 +0200 Subject: [PATCH 030/130] Implement non-linearity term for NDVI-weighted hybrid-green correction when converting NDVI to blend factor. --- satpy/composites/spectral.py | 24 ++++++++++++++++--- satpy/tests/compositor_tests/test_spectral.py | 13 ++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/satpy/composites/spectral.py b/satpy/composites/spectral.py index c0ccaff64f..744637551a 100644 --- a/satpy/composites/spectral.py +++ b/satpy/composites/spectral.py @@ -114,8 +114,8 @@ class NDVIHybridGreen(SpectralBlender): This green band correction follows the same approach as the HybridGreen compositor, but with a dynamic blend factor `f` that depends on the pixel-level Normalized Differece Vegetation Index (NDVI). The higher the NDVI, the - smaller the contribution from the nir channel will be, following a liner relationship between the two ranges - `[ndvi_min, ndvi_max]` and `limits`. + smaller the contribution from the nir channel will be, following a liner (default) or non-linear relationship + between the two ranges `[ndvi_min, ndvi_max]` and `limits`. As an example, a new green channel using e.g. FCI data and the NDVIHybridGreen compositor can be defined like:: @@ -124,6 +124,7 @@ class NDVIHybridGreen(SpectralBlender): ndvi_min: 0.0 ndvi_max: 1.0 limits: [0.15, 0.05] + strength: 1.0 prerequisites: - name: vis_05 modifiers: [sunz_corrected, rayleigh_corrected] @@ -138,17 +139,29 @@ class NDVIHybridGreen(SpectralBlender): pixels with NDVI=1.0 will be a weighted average with 5% contribution from the near-infrared vis_08 channel and the remaining 95% from the native green vis_05 channel. For other values of NDVI a linear interpolation between these values will be performed. + + A strength larger or smaller than 1.0 will introduce a non-linear relationship between the two ranges + `[ndvi_min, ndvi_max]` and `limits`. Hence, a higher strength (> 1.0) will result in a slower transition + to higher/lower fractions at the NDVI extremes. Similarly, a lower strength (< 1.0) will result in a + faster transition to higher/lower fractions at the NDVI extremes. """ - def __init__(self, *args, ndvi_min=0.0, ndvi_max=1.0, limits=(0.15, 0.05), **kwargs): + def __init__(self, *args, ndvi_min=0.0, ndvi_max=1.0, limits=(0.15, 0.05), strength=1.0, **kwargs): """Initialize class and set the NDVI limits and the corresponding blending fraction limits.""" + if strength <= 0.0: + raise ValueError(f"Expected stength greater than 0.0, got {strength}.") + self.ndvi_min = ndvi_min self.ndvi_max = ndvi_max self.limits = limits + self.strength = strength super().__init__(*args, **kwargs) def __call__(self, projectables, optional_datasets=None, **attrs): """Construct the hybrid green channel weighted by NDVI.""" + LOG.info(f"Applying NDVI-weighted hybrid-green correction with limits [{self.limits[0]}, " + f"{self.limits[1]}] and stength {self.strength}.") + ndvi_input = self.match_data_arrays([projectables[1], projectables[2]]) ndvi = (ndvi_input[1] - ndvi_input[0]) / (ndvi_input[1] + ndvi_input[0]) @@ -156,6 +169,11 @@ def __call__(self, projectables, optional_datasets=None, **attrs): ndvi.data = da.where(ndvi > self.ndvi_min, ndvi, self.ndvi_min) ndvi.data = da.where(ndvi < self.ndvi_max, ndvi, self.ndvi_max) + # Apply non-linearity to the ndvi for a non-linear conversion from ndvi to fraction. This can be used for a + # slower transision to higher/lower fractions at the ndvi extremes. If strength equals 1.0, this operation has + # no effect on ndvi. + ndvi = ndvi ** self.strength / (ndvi ** self.strength + (1 - ndvi) ** self.strength) + fraction = (ndvi - self.ndvi_min) / (self.ndvi_max - self.ndvi_min) * (self.limits[1] - self.limits[0]) \ + self.limits[0] self.fractions = (1 - fraction, fraction) diff --git a/satpy/tests/compositor_tests/test_spectral.py b/satpy/tests/compositor_tests/test_spectral.py index 2e9f59c13f..03e51a5043 100644 --- a/satpy/tests/compositor_tests/test_spectral.py +++ b/satpy/tests/compositor_tests/test_spectral.py @@ -77,6 +77,7 @@ def test_ndvi_hybrid_green(self): comp = NDVIHybridGreen('ndvi_hybrid_green', limits=(0.15, 0.05), prerequisites=(0.51, 0.65, 0.85), standard_name='toa_bidirectional_reflectance') + # Test General functionality with linear strength (=1.0) res = comp((self.c01, self.c02, self.c03)) assert isinstance(res, xr.DataArray) assert isinstance(res.data, da.Array) @@ -85,6 +86,18 @@ def test_ndvi_hybrid_green(self): data = res.values np.testing.assert_array_almost_equal(data, np.array([[0.2633, 0.3071], [0.2115, 0.3420]]), decimal=4) + # Test invalid strength + with pytest.raises(ValueError): + _ = NDVIHybridGreen('ndvi_hybrid_green', strength=0.0, prerequisites=(0.51, 0.65, 0.85), + standard_name='toa_bidirectional_reflectance') + + # Test non-linear strength + comp = NDVIHybridGreen('ndvi_hybrid_green', limits=(0.15, 0.05), strength=2.0, prerequisites=(0.51, 0.65, 0.85), + standard_name='toa_bidirectional_reflectance') + + res = comp((self.c01, self.c02, self.c03)) + np.testing.assert_array_almost_equal(res.values, np.array([[0.2646, 0.3075], [0.2120, 0.3471]]), decimal=4) + def test_green_corrector(self): """Test the deprecated class for green corrections.""" comp = GreenCorrector('blended_channel', fractions=(0.85, 0.15), prerequisites=(0.51, 0.85), From 5dc3d3edc53f2005cee3a3251e92c8fd913f6c28 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 6 Apr 2023 11:54:04 +0200 Subject: [PATCH 031/130] Remove monochromatic colorization and use available palettes functionality instead. --- satpy/composites/__init__.py | 11 ++------ satpy/enhancements/__init__.py | 43 +++++++---------------------- satpy/etc/composites/ahi.yaml | 3 +- satpy/etc/enhancements/generic.yaml | 4 +-- 4 files changed, 16 insertions(+), 45 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 995ef7d317..db120293ec 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1113,7 +1113,7 @@ class LowCloudCompositor(CloudCompositor): def __init__(self, name, values_land=(1,), values_sea=(0,), range_land=(1.0, 4.5), range_sea=(0.0, 4.0), - transition_gamma=1.0, color=(140.25, 191.25, 249.9), **kwargs): + transition_gamma=1.0, **kwargs): """Init info. Collect custom configuration values. @@ -1129,22 +1129,17 @@ def __init__(self, name, values_land=(1,), values_sea=(0,), transition_min values. transition_gamma (float): Gamma correction to apply to the alpha channel within the brightness temperature difference range. - color (list): RGB definition of color to use for the low-level clouds in the composite (the final - color will be a function of the corresponding trasnparency/alpha channel). """ if len(range_land) != 2: raise ValueError(f"Expected 2 `range_land` values, got {len(range_land)}") if len(range_sea) != 2: raise ValueError(f"Expected 2 `range_sea` values, got {len(range_sea)}") - if type(color) not in [list, tuple] or len(color) != 3: - raise ValueError("Expected list/tuple with the red, green and blue color components.") self.values_land = values_land if type(values_land) in [list, tuple] else [values_land] self.values_sea = values_sea if type(values_sea) in [list, tuple] else [values_sea] self.range_land = range_land self.range_sea = range_sea self.transition_gamma = transition_gamma - self.color = color self.transition_min = None # Placeholder for later use in CloudCompositor self.transition_max = None # Placeholder for later use in CloudCompositor super().__init__(name, transition_gamma=transition_gamma, **kwargs) @@ -1171,11 +1166,11 @@ def __call__(self, projectables, **kwargs): # Call CloudCompositor for land surface pixels self.transition_min, self.transition_max = self.range_land - res = super().__call__([btd.where(lsm.isin(self.values_land))], low_cloud_color=self.color, **kwargs) + res = super().__call__([btd.where(lsm.isin(self.values_land))], **kwargs) # Call CloudCompositor for sea/water surface pixels self.transition_min, self.transition_max = self.range_sea - res_sea = super().__call__([btd.where(lsm.isin(self.values_sea))], low_cloud_color=self.color, **kwargs) + res_sea = super().__call__([btd.where(lsm.isin(self.values_sea))], **kwargs) # Compine resutls for land and sea/water surface pixels res = res.where(lsm.isin(self.values_land), res_sea) diff --git a/satpy/enhancements/__init__.py b/satpy/enhancements/__init__.py index 0ae8fd45c0..29f2cbdf54 100644 --- a/satpy/enhancements/__init__.py +++ b/satpy/enhancements/__init__.py @@ -358,39 +358,16 @@ def _merge_colormaps(kwargs, img=None): from trollimage.colormap import Colormap full_cmap = None - # TODO - # - Improve check if both palettes and monochromatic are set - # - Improve exception handling for monochromatic cases - # - Resolve RunTimeWarnings - - if 'palettes' in kwargs: - palette = kwargs['palettes'] - if isinstance(palette, Colormap): - full_cmap = palette - else: - for itm in palette: - cmap = create_colormap(itm, img) - if full_cmap is None: - full_cmap = cmap - else: - full_cmap = full_cmap + cmap - - if 'monochromatic' in kwargs: - palette = {} - color = kwargs['monochromatic'].get('color', None) - if color is None: - # TODO: add error - pass - elif isinstance(color, (list, tuple)): - palette['colors'] = [color] - elif isinstance(color, str): - var = img.data.attrs.get(color, None) - if not isinstance(var, (tuple, list)): - # TODO: add error - pass - palette['colors'] = [var] - - full_cmap = create_colormap(palette, img) + palette = kwargs['palettes'] + if isinstance(palette, Colormap): + full_cmap = palette + else: + for itm in palette: + cmap = create_colormap(itm, img) + if full_cmap is None: + full_cmap = cmap + else: + full_cmap = full_cmap + cmap return full_cmap diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index f0543dd7d5..5c73eea7e9 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -539,7 +539,6 @@ composites: compositor: !!python/name:satpy.composites.LowCloudCompositor values_sea: 0 values_land: 100 - color: [140.25, 191.25, 249.9] prerequisites: - compositor: !!python/name:satpy.composites.DifferenceCompositor prerequisites: @@ -567,7 +566,7 @@ composites: geo_color_night_without_background: compositor: !!python/name:satpy.composites.BackgroundCompositor - standard_name: night_ir_with_background + standard_name: night_ir prerequisites: - geo_color_low_clouds - geo_color_high_clouds diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index 2094327918..362625fa15 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -866,8 +866,8 @@ enhancements: - name: colorize method: !!python/name:satpy.enhancements.colorize kwargs: - monochromatic: - color: low_cloud_color + palettes: + - {colors: [[140.25, 191.25, 249.9]]} colorized_ir_clouds: standard_name: colorized_ir_clouds From 2c0406010d49b69add2de73621c7b5c77adeb680 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Wed, 23 Aug 2023 16:19:14 +0200 Subject: [PATCH 032/130] Specify name of GeoColor enhancements. Add enhancement for GeoColor day-night blend. --- satpy/etc/enhancements/generic.yaml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index 362625fa15..fd2c02fe48 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -838,8 +838,8 @@ enhancements: kwargs: weight: 1.0 - ir_high_cloud: - standard_name: ir_high_cloud + geo_color_high_clouds: + standard_name: geo_color_high_clouds operations: - name: inverse method: !!python/name:satpy.enhancements.invert @@ -849,11 +849,9 @@ enhancements: method: !!python/name:satpy.enhancements.stretch kwargs: stretch: linear - - name: 3d - method: !!python/name:satpy.enhancements.three_d_effect - ir_low_cloud: - standard_name: ir_low_cloud + geo_color_low_clouds: + standard_name: geo_color_low_clouds operations: - name: inverse method: !!python/name:satpy.enhancements.invert @@ -869,6 +867,16 @@ enhancements: palettes: - {colors: [[140.25, 191.25, 249.9]]} + geo_color_day_night_blend: + standard_name: geo_color_day_night_blend + operations: + - name: stretch + method: !!python/name:satpy.enhancements.stretch + kwargs: + stretch: crude + min_stretch: [ 0,0,0 ] + max_stretch: [ 1,1,1 ] + colorized_ir_clouds: standard_name: colorized_ir_clouds operations: From 2c4f086e2a5d65ea4bf712bc745d930b611e5a7b Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Wed, 23 Aug 2023 16:20:15 +0200 Subject: [PATCH 033/130] Add FCI GeoColor recipes. --- satpy/etc/composites/fci.yaml | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 4e450a9779..4c091bd012 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -64,3 +64,50 @@ composites: - name: ndvi_hybrid_green_raw - name: vis_04 standard_name: true_color_raw + + # GeoColor + geo_color: + compositor: !!python/name:satpy.composites.DayNightCompositor + lim_low: 73 + lim_high: 82 + standard_name: geo_color_day_night_blend + prerequisites: + - true_color + - geo_color_night + + # GeoColor Night-time + geo_color_high_clouds: + standard_name: geo_color_high_clouds + compositor: !!python/name:satpy.composites.HighCloudCompositor + prerequisites: + - name: ir_105 + + geo_color_low_clouds: + standard_name: geo_color_low_clouds + compositor: !!python/name:satpy.composites.LowCloudCompositor + values_sea: 0 + values_land: 100 + prerequisites: + - compositor: !!python/name:satpy.composites.DifferenceCompositor + prerequisites: + - name: ir_105 + - name: ir_38 + - name: ir_105 + - compositor: !!python/name:satpy.composites.StaticImageCompositor + standard_name: land_sea_mask + # TODO Change filename + filename: "/tcenas/proj/optcalimg/strandgren/GeoColor/static_data/gshhs_land_sea_mask_3km_i.tif" + + geo_color_background_with_low_clouds: + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: night_ir_with_background + prerequisites: + - geo_color_low_clouds + - _night_background_hires + + geo_color_night: + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: night_ir_with_background + prerequisites: + - geo_color_high_clouds + - geo_color_background_with_low_clouds From b0a6172c39a7517b44569d4b0d46c20fb028135c Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Wed, 23 Aug 2023 16:29:41 +0200 Subject: [PATCH 034/130] Align AHI GeoColor recipes with FCI developments/recipes. --- satpy/etc/composites/ahi.yaml | 53 ++++++++--------------------------- 1 file changed, 11 insertions(+), 42 deletions(-) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 5c73eea7e9..c0008366cc 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -14,22 +14,6 @@ modifiers: - solar_azimuth_angle - solar_zenith_angle - geo_color_rayleigh_corrected: - modifier: !!python/name:satpy.modifiers.PSPRayleighReflectance - atmosphere: us-standard - aerosol_type: rayleigh_only - reduce_lim_low: 70 - reduce_lim_high: 105 - reduce_strength: 1.5 - prerequisites: - - name: B03 - modifiers: [sunz_corrected] - optional_prerequisites: - - satellite_azimuth_angle - - satellite_zenith_angle - - solar_azimuth_angle - - solar_zenith_angle - composites: green: deprecation_warning: "'green' is a deprecated composite. Use the equivalent 'hybrid_green' instead." @@ -488,30 +472,22 @@ composites: geo_color: compositor: !!python/name:satpy.composites.DayNightCompositor - lim_low: 80 - lim_high: 88 - standard_name: true_color_with_night_ir + lim_low: 73 + lim_high: 82 + standard_name: geo_color_day_night_blend prerequisites: - geo_color_true_color - geo_color_night - geo_color_without_background: - compositor: !!python/name:satpy.composites.DayNightCompositor - lim_low: 80 - lim_high: 88 - standard_name: true_color_with_night_ir - prerequisites: - - geo_color_true_color - - geo_color_night_without_background - # GeoColor Daytime geo_color_green: compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen + limits: [0.15, 0.05] prerequisites: - name: B02 - modifiers: [ sunz_corrected, geo_color_rayleigh_corrected ] + modifiers: [ sunz_corrected, rayleigh_corrected ] - name: B03 - modifiers: [ sunz_corrected, geo_color_rayleigh_corrected ] + modifiers: [ sunz_corrected, rayleigh_corrected ] - name: B04 modifiers: [ sunz_corrected ] standard_name: toa_bidirectional_reflectance @@ -520,22 +496,22 @@ composites: compositor: !!python/name:satpy.composites.SelfSharpenedRGB prerequisites: - name: B03 - modifiers: [ sunz_corrected, geo_color_rayleigh_corrected ] + modifiers: [ sunz_corrected, rayleigh_corrected ] - name: geo_color_green - name: B01 - modifiers: [ sunz_corrected, geo_color_rayleigh_corrected ] + modifiers: [ sunz_corrected, rayleigh_corrected ] high_resolution_band: red standard_name: true_color # GeoColor Night-time geo_color_high_clouds: - standard_name: ir_high_cloud + standard_name: geo_color_high_clouds compositor: !!python/name:satpy.composites.HighCloudCompositor prerequisites: - name: B13 geo_color_low_clouds: - standard_name: ir_low_cloud + standard_name: geo_color_low_clouds compositor: !!python/name:satpy.composites.LowCloudCompositor values_sea: 0 values_land: 100 @@ -548,7 +524,7 @@ composites: - compositor: !!python/name:satpy.composites.StaticImageCompositor standard_name: land_sea_mask # TODO Change filename - filename: "/tcenas/scratch/strandgren/GeoColor/gshhs_land_sea_mask_3km_i.tif" + filename: "/tcenas/proj/optcalimg/strandgren/GeoColor/static_data/gshhs_land_sea_mask_3km_i.tif" geo_color_background_with_low_clouds: compositor: !!python/name:satpy.composites.BackgroundCompositor @@ -563,10 +539,3 @@ composites: prerequisites: - geo_color_high_clouds - geo_color_background_with_low_clouds - - geo_color_night_without_background: - compositor: !!python/name:satpy.composites.BackgroundCompositor - standard_name: night_ir - prerequisites: - - geo_color_low_clouds - - geo_color_high_clouds From c8aa8aeebce0e1fbba99d4d6117582d0bb0792de Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Wed, 23 Aug 2023 16:47:54 +0200 Subject: [PATCH 035/130] Modify thresholds of FCI GeoColor low cloud detection in order to reduce false alarms with early FCI data. --- satpy/etc/composites/fci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 4c091bd012..caa8858734 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -87,6 +87,8 @@ composites: compositor: !!python/name:satpy.composites.LowCloudCompositor values_sea: 0 values_land: 100 + range_land: [4.35, 6.75] + range_sea: [1.35, 5.0] prerequisites: - compositor: !!python/name:satpy.composites.DifferenceCompositor prerequisites: From 813845f863a0ddfd61b85e24978262f7a0183166 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 24 Aug 2023 17:16:17 +0200 Subject: [PATCH 036/130] Restore NDVI true color recipe and use in GeoColor. --- satpy/etc/composites/ahi.yaml | 48 +++++++++++++++++------------------ 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index c0008366cc..a4e0994a9c 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -103,6 +103,17 @@ composites: - wavelength: 0.85 standard_name: toa_reflectance + ndvi_hybrid_green: + compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen + prerequisites: + - name: B02 + modifiers: [ sunz_corrected, rayleigh_corrected ] + - name: B03 + modifiers: [ sunz_corrected, rayleigh_corrected ] + - name: B04 + modifiers: [ sunz_corrected ] + standard_name: toa_bidirectional_reflectance + airmass: # PDF slides: https://www.eumetsat.int/website/home/News/ConferencesandEvents/DAT_2833302.html # Under session 2 by Akihiro Shimizu (JMA) @@ -260,6 +271,17 @@ composites: high_resolution_band: red standard_name: true_color + true_color_ndvi_green: + compositor: !!python/name:satpy.composites.SelfSharpenedRGB + prerequisites: + - name: B03 + modifiers: [ sunz_corrected, rayleigh_corrected ] + - name: ndvi_hybrid_green + - name: B01 + modifiers: [ sunz_corrected, rayleigh_corrected ] + high_resolution_band: red + standard_name: true_color + natural_color_nocorr: compositor: !!python/name:satpy.composites.SelfSharpenedRGB prerequisites: @@ -476,33 +498,9 @@ composites: lim_high: 82 standard_name: geo_color_day_night_blend prerequisites: - - geo_color_true_color + - true_color_ndvi_green - geo_color_night - # GeoColor Daytime - geo_color_green: - compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen - limits: [0.15, 0.05] - prerequisites: - - name: B02 - modifiers: [ sunz_corrected, rayleigh_corrected ] - - name: B03 - modifiers: [ sunz_corrected, rayleigh_corrected ] - - name: B04 - modifiers: [ sunz_corrected ] - standard_name: toa_bidirectional_reflectance - - geo_color_true_color: - compositor: !!python/name:satpy.composites.SelfSharpenedRGB - prerequisites: - - name: B03 - modifiers: [ sunz_corrected, rayleigh_corrected ] - - name: geo_color_green - - name: B01 - modifiers: [ sunz_corrected, rayleigh_corrected ] - high_resolution_band: red - standard_name: true_color - # GeoColor Night-time geo_color_high_clouds: standard_name: geo_color_high_clouds From 70f30e68bd539e113e2555a7c9bc0b99c2f2b69d Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Fri, 25 Aug 2023 12:18:40 +0200 Subject: [PATCH 037/130] Add Geocolor recipes for ABI. --- satpy/etc/composites/abi.yaml | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/satpy/etc/composites/abi.yaml b/satpy/etc/composites/abi.yaml index 489ef1f210..2f5d68e09e 100644 --- a/satpy/etc/composites/abi.yaml +++ b/satpy/etc/composites/abi.yaml @@ -705,3 +705,50 @@ composites: prerequisites: - name: C14 standard_name: highlighted_toa_brightness_temperature + + # GeoColor + geo_color: + compositor: !!python/name:satpy.composites.DayNightCompositor + lim_low: 73 + lim_high: 82 + standard_name: geo_color_day_night_blend + prerequisites: + - true_color + - geo_color_night + + # GeoColor Night-time + geo_color_high_clouds: + standard_name: geo_color_high_clouds + compositor: !!python/name:satpy.composites.HighCloudCompositor + prerequisites: + - name: C13 + + geo_color_low_clouds: + standard_name: geo_color_low_clouds + compositor: !!python/name:satpy.composites.LowCloudCompositor + values_sea: 0 + values_land: 100 + prerequisites: + - compositor: !!python/name:satpy.composites.DifferenceCompositor + prerequisites: + - name: C13 + - name: C07 + - name: C13 + - compositor: !!python/name:satpy.composites.StaticImageCompositor + standard_name: land_sea_mask + # TODO Change filename + filename: "/tcenas/proj/optcalimg/strandgren/GeoColor/static_data/gshhs_land_sea_mask_3km_i.tif" + + geo_color_background_with_low_clouds: + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: night_ir_with_background + prerequisites: + - geo_color_low_clouds + - _night_background_hires + + geo_color_night: + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: night_ir_with_background + prerequisites: + - geo_color_high_clouds + - geo_color_background_with_low_clouds From 78c11911769be180470eb8da8f131d8fea4ad333 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Tue, 7 Nov 2023 08:10:07 +0100 Subject: [PATCH 038/130] Update thresholds for high-level and low-level cloud layers abased on feedback from CIRA, i.e. the developers of the GeoColor composite blend (personal communication, 27.09.2023). --- satpy/composites/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 5924d7794c..0deecec642 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1077,7 +1077,7 @@ class HighCloudCompositor(CloudCompositor): of where abs(latitude). """ - def __init__(self, name, transition_min=(200., 220.), transition_max=280, latitude_min=(30., 60.), + def __init__(self, name, transition_min=(210., 230.), transition_max=300, latitude_min=(30., 60.), transition_gamma=1.0, **kwargs): """Collect custom configuration values. @@ -1149,7 +1149,7 @@ class LowCloudCompositor(CloudCompositor): """ def __init__(self, name, values_land=(1,), values_sea=(0,), - range_land=(1.0, 4.5), + range_land=(0.0, 4.0), range_sea=(0.0, 4.0), transition_gamma=1.0, **kwargs): """Init info. From 68f93047257267520f73d273b7eb057c63e00ab1 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Tue, 7 Nov 2023 08:11:47 +0100 Subject: [PATCH 039/130] Add TODOs for code consolidation and optimization. --- satpy/composites/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 0deecec642..dea3830b22 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1109,6 +1109,8 @@ def __call__(self, projectables, **kwargs): `projectables` is expected to be a list or tuple with a single element: - index 0: Brightness temperature of a thermal infrared window channel (e.g. 10.5 microns). """ + # TODO Optimize and make sure that there are no early unnecessary dask computations. Is there a way to avoid + # computation of the latitude array? if len(projectables) != 1: raise ValueError(f"Expected 1 dataset, got {len(projectables)}") @@ -1191,6 +1193,7 @@ def __call__(self, projectables, **kwargs): - index 1. Brightness temperature of the window channel (used to filter out noise-induced false alarms). - index 2: Land-Sea-Mask. """ + # TODO Optimize and make sure that there are no early unnecessary dask computations if len(projectables) != 3: raise ValueError(f"Expected 3 datasets, got {len(projectables)}") @@ -1200,6 +1203,8 @@ def __call__(self, projectables, **kwargs): lsm = lsm.round() # Make sure to have whole numbers in case of smearing from resampling # Avoid spurious false alarms caused by noise in the 3.9um channel that can occur for very cold cloud tops + # TODO Consolidate this. Should it really be set to zero and thus within the threshold range? What if the + # lower threshold would be changed to -1 btd = btd.where(bt_win >= 230, 0.0) # Call CloudCompositor for land surface pixels From 07d10c9915928cf6d43794db4a89bfbf43fc2d7b Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Tue, 7 Nov 2023 09:04:49 +0100 Subject: [PATCH 040/130] Add url to land water mask to use for GeoColor low-level cloud detection. --- satpy/etc/composites/abi.yaml | 4 ++-- satpy/etc/composites/ahi.yaml | 4 ++-- satpy/etc/composites/fci.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/satpy/etc/composites/abi.yaml b/satpy/etc/composites/abi.yaml index a686b829ae..e950ba027f 100644 --- a/satpy/etc/composites/abi.yaml +++ b/satpy/etc/composites/abi.yaml @@ -783,8 +783,8 @@ composites: - name: C13 - compositor: !!python/name:satpy.composites.StaticImageCompositor standard_name: land_sea_mask - # TODO Change filename - filename: "/tcenas/proj/optcalimg/strandgren/GeoColor/static_data/gshhs_land_sea_mask_3km_i.tif" + url: "https://zenodo.org/records/10076199/files/gshhs_land_water_mask_3km_i.tif" + known_hash: "sha256:96df83c57416217e191f95dde3d3c1ce0373a8fc220e929228873db246ca3569" geo_color_background_with_low_clouds: compositor: !!python/name:satpy.composites.BackgroundCompositor diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 4f6ae6932f..e088bcf1a6 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -533,8 +533,8 @@ composites: - name: B13 - compositor: !!python/name:satpy.composites.StaticImageCompositor standard_name: land_sea_mask - # TODO Change filename - filename: "/tcenas/proj/optcalimg/strandgren/GeoColor/static_data/gshhs_land_sea_mask_3km_i.tif" + url: "https://zenodo.org/records/10076199/files/gshhs_land_water_mask_3km_i.tif" + known_hash: "sha256:96df83c57416217e191f95dde3d3c1ce0373a8fc220e929228873db246ca3569" geo_color_background_with_low_clouds: compositor: !!python/name:satpy.composites.BackgroundCompositor diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 25ba032cac..c8a32910ca 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -130,8 +130,8 @@ composites: - name: ir_105 - compositor: !!python/name:satpy.composites.StaticImageCompositor standard_name: land_sea_mask - # TODO Change filename - filename: "/tcenas/proj/optcalimg/strandgren/GeoColor/static_data/gshhs_land_sea_mask_3km_i.tif" + url: "https://zenodo.org/records/10076199/files/gshhs_land_water_mask_3km_i.tif" + known_hash: "sha256:96df83c57416217e191f95dde3d3c1ce0373a8fc220e929228873db246ca3569" geo_color_background_with_low_clouds: compositor: !!python/name:satpy.composites.BackgroundCompositor From 39bf45e9dd701339fec9531b4b9fc50b7be5ade4 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Tue, 7 Nov 2023 09:07:46 +0100 Subject: [PATCH 041/130] Update doc strings. --- satpy/composites/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index dea3830b22..bc5a199aa0 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1063,7 +1063,8 @@ class HighCloudCompositor(CloudCompositor): temperature (cloud opacity). In contrast to the `CloudCompositor`, the brightness temperature threshold at the lower end, used to identify high opaque clouds, is made a function of the latitude in order to have tropopause level clouds appear opaque at both high and low latitudes. This follows the Geocolor - implementation of high clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`). + implementation of high clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`), but + with some adjustments to the thresholds based on recent developments and feedback from CIRA. The two brightness temperature thresholds in `transition_min` are used together with the corresponding latitude limits in `latitude_min` to compute a modified version of `transition_min` that is later used @@ -1144,7 +1145,8 @@ class LowCloudCompositor(CloudCompositor): function of the `BTD` value itself. Two sets of thresholds are used, one set for land surface types (`range_land`) and another one for sea/water surface types (`range_sea`), respectively. Hence, this compositor requires a land-sea-mask as a prerequisite input. This follows the GeoColor - implementation of night-time low-level clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`). + implementation of night-time low-level clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`), but + with some adjustments to the thresholds based on recent developments and feedback from CIRA. Please note that the spectral test and thus the output of the compositor (using the expected input data) is only applicable during night-time. @@ -1165,8 +1167,6 @@ def __init__(self, name, values_land=(1,), values_sea=(0,), difference over land surface types. range_sea (tuple): Threshold values used for masking low-level clouds from the brightness temperature difference over sea/water. - latitude_min (tuple): Latitude values defining the intervals for computing latitude-dependent - transition_min values. transition_gamma (float): Gamma correction to apply to the alpha channel within the brightness temperature difference range. """ From 60a7c1b3a527b896838114a5ee0f2ce7b26fabec Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 8 Dec 2023 09:42:53 +0200 Subject: [PATCH 042/130] Fix RealisticColors compositor not to upcast data to float64 --- satpy/composites/__init__.py | 8 ++++---- satpy/tests/test_composites.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 9295f94dc7..686b8c4c27 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -992,17 +992,17 @@ def __call__(self, projectables, *args, **kwargs): hrv = projectables[2] try: - ch3 = 3 * hrv - vis06 - vis08 + ch3 = 3.0 * hrv - vis06 - vis08 ch3.attrs = hrv.attrs except ValueError: raise IncompatibleAreas ndvi = (vis08 - vis06) / (vis08 + vis06) - ndvi = np.where(ndvi < 0, 0, ndvi) + ndvi = ndvi.where(ndvi >= 0.0, 0.0) - ch1 = ndvi * vis06 + (1 - ndvi) * vis08 + ch1 = ndvi * vis06 + (1.0 - ndvi) * vis08 ch1.attrs = vis06.attrs - ch2 = ndvi * vis08 + (1 - ndvi) * vis06 + ch2 = ndvi * vis08 + (1.0 - ndvi) * vis06 ch2.attrs = vis08.attrs res = super(RealisticColors, self).__call__((ch1, ch2, ch3), diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index 7fbe177bfb..4a7b2a2ce9 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -1867,3 +1867,37 @@ def _create_fake_composite_config(yaml_filename: str): }, comp_file, ) + + +class TestRealisticColors: + """Test the SEVIRI Realistic Colors compositor.""" + + def test_realistic_colors(self): + """Test the compositor.""" + from satpy.composites import RealisticColors + + vis06 = xr.DataArray(da.arange(0, 15, dtype=np.float32).reshape(3, 5), dims=("y", "x"), + attrs={"foo": "foo"}) + vis08 = xr.DataArray(da.arange(15, 0, -1, dtype=np.float32).reshape(3, 5), dims=("y", "x"), + attrs={"bar": "bar"}) + hrv = xr.DataArray(6 * da.ones((3, 5), dtype=np.float32), dims=("y", "x"), + attrs={"baz": "baz"}) + + expected_red = np.array([[0.0, 2.733333, 4.9333334, 6.6, 7.733333], + [8.333333, 8.400001, 7.9333334, 7.0, 6.0], + [5.0, 4.0, 3.0, 2.0, 1.0]], dtype=np.float32) + expected_green = np.array([ + [15.0, 12.266666, 10.066668, 8.400001, 7.2666664], + [6.6666665, 6.6000004, 7.0666666, 8.0, 9.0], + [10.0, 11.0, 12.0, 13.0, 14.0]], dtype=np.float32) + + with dask.config.set(scheduler=CustomScheduler(max_computes=1)): + comp = RealisticColors("Ni!") + res = comp((vis06, vis08, hrv)) + + arr = res.values + + assert res.dtype == np.float32 + np.testing.assert_allclose(arr[0, :, :], expected_red) + np.testing.assert_allclose(arr[1, :, :], expected_green) + np.testing.assert_allclose(arr[2, :, :], 3.0) From 91b59ca721cdda8d8b7cf4716c74050aa589d06b Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 8 Dec 2023 10:07:50 -0600 Subject: [PATCH 043/130] Workaround AWIPS bug not handling integers properly in "awips_tiled" writer --- satpy/tests/writer_tests/test_awips_tiled.py | 2 +- satpy/writers/awips_tiled.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/satpy/tests/writer_tests/test_awips_tiled.py b/satpy/tests/writer_tests/test_awips_tiled.py index 63113a9f94..dbc1bc82d7 100644 --- a/satpy/tests/writer_tests/test_awips_tiled.py +++ b/satpy/tests/writer_tests/test_awips_tiled.py @@ -198,7 +198,7 @@ def test_basic_numbered_1_tile(self, extra_attrs, expected_filename, use_save_da check_required_properties(unmasked_ds, output_ds) scale_factor = output_ds["data"].encoding["scale_factor"] np.testing.assert_allclose(input_data_arr.values, output_ds["data"].data, - atol=scale_factor / 2) + atol=scale_factor * 0.75) def test_units_length_warning(self, tmp_path): """Test long 'units' warnings are raised.""" diff --git a/satpy/writers/awips_tiled.py b/satpy/writers/awips_tiled.py index 15680e8091..03ce3e9d68 100644 --- a/satpy/writers/awips_tiled.py +++ b/satpy/writers/awips_tiled.py @@ -630,7 +630,13 @@ def _get_factor_offset_fill(input_data_arr, vmin, vmax, encoding): # max value fills = [2 ** (file_bit_depth - 1) - 1] - mx = (vmax - vmin) / (2 ** bit_depth - 1 - num_fills) + # NOTE: AWIPS is buggy and does not properly handle both + # halves an integers data space. The below code limits + # unsigned integers to the positive half and this seems + # to work better with current AWIPS. + mx = (vmax - vmin) / (2 ** (bit_depth - 1) - 1 - num_fills) + # NOTE: This is what the line should look like if AWIPS wasn't buggy: + # mx = (vmax - vmin) / (2 ** bit_depth - 1 - num_fills) bx = vmin if not is_unsigned and not unsigned_in_signed: bx += 2 ** (bit_depth - 1) * mx From aa98fd6d69335a8c862570e1ec3479d92f0cd9eb Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Mon, 11 Dec 2023 15:14:50 +0200 Subject: [PATCH 044/130] Update xarray version in compression tests for compression kwarg --- satpy/tests/writer_tests/test_cf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/writer_tests/test_cf.py b/satpy/tests/writer_tests/test_cf.py index 7fabb04f10..62c9995cde 100644 --- a/satpy/tests/writer_tests/test_cf.py +++ b/satpy/tests/writer_tests/test_cf.py @@ -570,5 +570,5 @@ def _should_use_compression_keyword(): versions = _get_backend_versions() return ( versions["libnetcdf"] >= Version("4.9.0") and - versions["xarray"] >= Version("2023.12") + versions["xarray"] >= Version("2024.1") ) From 77830d67ca86bfb398fa38ec10d21f097b2faffe Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Mon, 11 Dec 2023 13:39:02 +0000 Subject: [PATCH 045/130] Add AMV/AMVI file pattern and variable --- satpy/etc/readers/fci_l2_nc.yaml | 267 +++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) diff --git a/satpy/etc/readers/fci_l2_nc.yaml b/satpy/etc/readers/fci_l2_nc.yaml index 20d9935682..7c0724b6ac 100644 --- a/satpy/etc/readers/fci_l2_nc.yaml +++ b/satpy/etc/readers/fci_l2_nc.yaml @@ -64,6 +64,16 @@ file_types: file_patterns: - '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+FCI-2-ASR-{subtype}-{coverage}-{subsetting}-{component1}-{component2}-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' + nc_fci_amvi: + file_reader: !!python/name:readers.fci_amv_l2_nc.FciAmvL2NCFileHandler + file_patterns: + - '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+FCI-2-AMVI-{channel}-{coverage}-{subsetting}-{component1}-{component2}-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' + + nc_fci_amv: + file_reader: !!python/name:readers.fci_amv_l2_nc.FciAmvL2NCFileHandler + file_patterns: + - '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+FCI-2-AMV-{channel}-{coverage}-{subsetting}-{component1}-{component2}-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' + datasets: # CLM @@ -2734,3 +2744,260 @@ datasets: file_type: nc_fci_asr file_key: product_timeliness long_name: product_timeliness_index + +# AMV Intermediate Product + intm_latitude: + name: intm_latitude + file_type: nc_fci_amvi + file_key: intm_latitude + standard_name: latitude + + intm_longitude: + name: intm_longitude + file_type: nc_fci_amvi + file_key: intm_longitude + standard_name: longitude + + intm_speed: + name: intm_speed + file_type: nc_fci_amvi + file_key: intm_speed + standard_name: wind_speed + coordinates: + - intm_longitude + - intm_latitude + + intm_u_component: + name: intm_u_component + file_type: nc_fci_amvi + file_key: intm_u_component + standard_name: wind_speed_horizontal_component + coordinates: + - intm_longitude + - intm_latitude + + intm_v_component: + name: intm_v_component + file_type: nc_fci_amvi + file_key: intm_v_component + standard_name: wind_speed_vertical_component + coordinates: + - intm_longitude + - intm_latitude + + intm_direction: + name: intm_direction + file_type: nc_fci_amvi + file_key: intm_direction + standard_name: wind_to_direction + coordinates: + - intm_longitude + - intm_latitude + + intm_pressure: + name: intm_pressure + file_type: nc_fci_amvi + file_key: intm_pressure + standard_name: wind_pressure + coordinates: + - intm_longitude + - intm_latitude + + intm_temperature: + name: intm_temperature + file_type: nc_fci_amvi + file_key: intm_temperature + standard_name: wind_temperature + coordinates: + - intm_longitude + - intm_latitude + + intm_target_type: + name: intm_target_type + file_type: nc_fci_amvi + file_key: target_type + standard_name: wind_target_type + coordinates: + - intm_longitude + - intm_latitude + + intm_wind_method: + name: intm_wind_method + file_type: nc_fci_amvi + file_key: wind_method + standard_name: wind_wind_method + coordinates: + - intm_longitude + - intm_latitude + +# AMV Final Product + channel_id: + name: channel_id + file_type: nc_fci_amv + file_key: channel_id + standard_name: channel_id + + latitude: + name: latitude + file_type: nc_fci_amv + file_key: latitude + standard_name: latitude + + longitude: + name: longitude + file_type: nc_fci_amv + file_key: longitude + standard_name: longitude + + speed: + name: speed + file_type: nc_fci_amv + file_key: speed + standard_name: wind_speed + coordinates: + - longitude + - latitude + + speed_u_component: + name: speed_u_component + file_type: nc_fci_amv + file_key: speed_u_component + standard_name: wind_speed_horizontal_component + coordinates: + - longitude + - latitude + + speed_v_component: + name: speed_v_component + file_type: nc_fci_amv + file_key: speed_v_component + standard_name: wind_speed_vertical_component + coordinates: + - longitude + - latitude + + direction: + name: direction + file_type: nc_fci_amv + file_key: direction + standard_name: wind_to_direction + coordinates: + - longitude + - latitude + + pressure: + name: pressure + file_type: nc_fci_amv + file_key: pressure + standard_name: wind_pressure + coordinates: + - longitude + - latitude + + temperature: + name: temperature + file_type: nc_fci_amv + file_key: temperature + standard_name: wind_temperature + coordinates: + - longitude + - latitude + + target_type: + name: target_type + file_type: nc_fci_amv + file_key: target_type + standard_name: wind_target_type + coordinates: + - longitude + - latitude + + wind_method: + name: wind_method + file_type: nc_fci_amv + file_key: wind_method + standard_name: wind_wind_method + coordinates: + - longitude + - latitude + + fcst_u: + name: fcst_u + file_type: nc_fci_amv + file_key: forecast_u_component + standard_name: wind_forecast_u_component + coordinates: + - longitude + - latitude + + fcst_v: + name: fcst_v + file_type: nc_fci_amv + file_key: forecast_v_component + standard_name: wind_forecast_v_component + coordinates: + - longitude + - latitude + + best_fit_pres: + name: best_fit_pres + file_type: nc_fci_amv + file_key: best_fit_pressure + standard_name: wind_best_fit_pressure + coordinates: + - longitude + - latitude + + best_fit_u: + name: best_fit_u + file_type: nc_fci_amv + file_key: best_fit_u_component + standard_name: wind_best_fit_u_component + coordinates: + - longitude + - latitude + + best_fit_v: + name: best_fit_v + file_type: nc_fci_amv + file_key: best_fit_v_component + standard_name: wind_best_fit_v_component + coordinates: + - longitude + - latitude + + qi: + name: qi + file_type: nc_fci_amv + file_key: overall_reliability + standard_name: wind_overall_reliability + coordinates: + - longitude + - latitude + + qi_excl_fcst: + name: qi_excl_fcst + file_type: nc_fci_amv + file_key: overall_reliability_exc_forecast + standard_name: wind_overall_reliability_exc_forecast + coordinates: + - longitude + - latitude + + product_quality: + name: product_quality + file_type: nc_fci_amv + file_key: product_quality + long_name: product_quality_index + + product_completeness: + name: product_completeness + file_type: nc_fci_amv + file_key: product_completeness + long_name: product_completeness_index + + product_timeliness: + name: product_timeliness + file_type: nc_fci_amv + file_key: product_timeliness + long_name: product_timeliness_index From a43187efd63b48f68fe43a08c779c011fc0d7776 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Mon, 11 Dec 2023 16:19:14 +0200 Subject: [PATCH 046/130] Convert times in SEVIRI readers to nanosecond precision to silence warnings --- satpy/readers/seviri_base.py | 4 +++- satpy/tests/reader_tests/test_seviri_base.py | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/satpy/readers/seviri_base.py b/satpy/readers/seviri_base.py index 25e6ed1a8b..5b19e56833 100644 --- a/satpy/readers/seviri_base.py +++ b/satpy/readers/seviri_base.py @@ -475,8 +475,10 @@ def get_cds_time(days, msecs): days = np.array([days], dtype="int64") msecs = np.array([msecs], dtype="int64") + # use nanosecond precision to silence warning from XArray + nsecs = 1000000 * msecs.astype("timedelta64[ns]") time = np.datetime64("1958-01-01").astype("datetime64[ms]") + \ - days.astype("timedelta64[D]") + msecs.astype("timedelta64[ms]") + days.astype("timedelta64[D]") + nsecs time[time == np.datetime64("1958-01-01 00:00")] = np.datetime64("NaT") if len(time) == 1: diff --git a/satpy/tests/reader_tests/test_seviri_base.py b/satpy/tests/reader_tests/test_seviri_base.py index c2d190e084..42b79ea0c8 100644 --- a/satpy/tests/reader_tests/test_seviri_base.py +++ b/satpy/tests/reader_tests/test_seviri_base.py @@ -74,23 +74,28 @@ def test_chebyshev(self): exp = chebyshev4(coefs, time, domain) np.testing.assert_allclose(res, exp) - def test_get_cds_time(self): - """Test the get_cds_time function.""" - # Scalar + def test_get_cds_time_scalar(self): + """Test the get_cds_time function for scalar inputs.""" assert get_cds_time(days=21246, msecs=12 * 3600 * 1000) == np.datetime64("2016-03-03 12:00") - # Array + def test_get_cds_time_array(self): + """Test the get_cds_time function for array inputs.""" days = np.array([21246, 21247, 21248]) msecs = np.array([12*3600*1000, 13*3600*1000 + 1, 14*3600*1000 + 2]) expected = np.array([np.datetime64("2016-03-03 12:00:00.000"), np.datetime64("2016-03-04 13:00:00.001"), np.datetime64("2016-03-05 14:00:00.002")]) - np.testing.assert_equal(get_cds_time(days=days, msecs=msecs), expected) + res = get_cds_time(days=days, msecs=msecs) + np.testing.assert_equal(res, expected) + def test_get_cds_time_nanoseconds(self): + """Test the get_cds_time function for having nanosecond precision.""" days = 21246 - msecs = 12*3600*1000 + msecs = 12 * 3600 * 1000 expected = np.datetime64("2016-03-03 12:00:00.000") - np.testing.assert_equal(get_cds_time(days=days, msecs=msecs), expected) + res = get_cds_time(days=days, msecs=msecs) + np.testing.assert_equal(res, expected) + assert ".000000000" in res.__repr__() def test_pad_data_horizontally_bad_shape(self): """Test the error handling for the horizontal hrv padding.""" From df26a586eff71ed269e449dd8c59ef12a4b129a6 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Mon, 11 Dec 2023 18:34:06 +0200 Subject: [PATCH 047/130] Check for dtype instead of string representation --- satpy/tests/reader_tests/test_seviri_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_seviri_base.py b/satpy/tests/reader_tests/test_seviri_base.py index 42b79ea0c8..86f684bb5e 100644 --- a/satpy/tests/reader_tests/test_seviri_base.py +++ b/satpy/tests/reader_tests/test_seviri_base.py @@ -95,7 +95,7 @@ def test_get_cds_time_nanoseconds(self): expected = np.datetime64("2016-03-03 12:00:00.000") res = get_cds_time(days=days, msecs=msecs) np.testing.assert_equal(res, expected) - assert ".000000000" in res.__repr__() + assert res.dtype == np.dtype("datetime64[ns]") def test_pad_data_horizontally_bad_shape(self): """Test the error handling for the horizontal hrv padding.""" From 101c44ddf8b9da6101eaa13b0a902ae5574de3b4 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 11 Dec 2023 10:46:40 -0600 Subject: [PATCH 048/130] Add remaining JPSS satellite platform aliases to "mirs" reader --- satpy/readers/mirs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 1ee0912b0f..34edd02739 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -50,6 +50,10 @@ PLATFORMS = {"n18": "NOAA-18", "n19": "NOAA-19", "np": "NOAA-19", + "n20": "NOAA-20", + "n21": "NOAA-21", + "n22": "NOAA-22", + "n23": "NOAA-23", "m2": "MetOp-A", "m1": "MetOp-B", "m3": "MetOp-C", @@ -60,11 +64,14 @@ "f17": "DMSP-F17", "f18": "DMSP-F18", "gpm": "GPM", - "n20": "NOAA-20", } SENSOR = {"n18": amsu, "n19": amsu, "n20": "atms", + "n21": "atms", + "n22": "atms", + "n23": "atms", + "n24": "atms", "np": amsu, "m1": amsu, "m2": amsu, From 943085f54b14dfadee4ecd591d9321a3aeaf61aa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 20:24:37 +0000 Subject: [PATCH 049/130] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.6 → v0.1.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.6...v0.1.7) - [github.com/PyCQA/bandit: 1.7.5 → 1.7.6](https://github.com/PyCQA/bandit/compare/1.7.5...1.7.6) - [github.com/pycqa/isort: 5.12.0 → 5.13.1](https://github.com/pycqa/isort/compare/5.12.0...5.13.1) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99e77cb56a..8036f793f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ fail_fast: false repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.6' + rev: 'v0.1.7' hooks: - id: ruff - repo: https://github.com/pre-commit/pre-commit-hooks @@ -14,7 +14,7 @@ repos: - id: check-yaml args: [--unsafe] - repo: https://github.com/PyCQA/bandit - rev: '1.7.5' # Update me! + rev: '1.7.6' # Update me! hooks: - id: bandit args: [--ini, .bandit] @@ -29,7 +29,7 @@ repos: - types-requests args: ["--python-version", "3.9", "--ignore-missing-imports"] - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.1 hooks: - id: isort language_version: python3 From 8428da1e67a8befe4b79df0572426ddafb2d5585 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 11 Dec 2023 14:52:53 -0600 Subject: [PATCH 050/130] Change pre-commit update schedule to monthly --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8036f793f5..a398bd445f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,4 +36,5 @@ repos: ci: # To trigger manually, comment on a pull request with "pre-commit.ci autofix" autofix_prs: false + autoupdate_schedule: "monthly" skip: [bandit] From b8fea39fee9bfafc68039e1481784d2f9a440b3f Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 11 Dec 2023 14:53:24 -0600 Subject: [PATCH 051/130] Change dependabot to monthly updates --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 90e05c40d0..95179b06c9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,4 +8,4 @@ updates: - package-ecosystem: "github-actions" # See documentation for possible values directory: "/" # Location of package manifests schedule: - interval: "weekly" + interval: "monthly" From 8c7999539d095e2817ff4c734264b201c3e3e16c Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 11 Dec 2023 15:29:56 -0600 Subject: [PATCH 052/130] Update MiRS reader coefficient files to newer version --- satpy/etc/readers/mirs.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/satpy/etc/readers/mirs.yaml b/satpy/etc/readers/mirs.yaml index 4e70fbed2c..5ca15f66b0 100644 --- a/satpy/etc/readers/mirs.yaml +++ b/satpy/etc/readers/mirs.yaml @@ -8,13 +8,13 @@ reader: reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [amsu, amsu-mhs, atms, ssmis, gmi] data_files: - - url: "https://zenodo.org/record/4472664/files/limbcoef_atmsland_noaa20.txt" - known_hash: "08a3b7c1594a963610dd864b7ecd12f0ab486412d35185c2371d924dd92c5779" - - url: "https://zenodo.org/record/4472664/files/limbcoef_atmsland_snpp.txt" + - url: "https://zenodo.org/record/10357932/files/limbcoef_atmsland_noaa20.txt" + known_hash: "08deca15afe8638effac9e6ccb442c2c386f5444926129d30a250d5840264c1d" + - url: "https://zenodo.org/record/10357932/files/limbcoef_atmsland_snpp.txt" known_hash: "4b01543699792306711ef1699244e96186487e8a869e4ae42bf1f0e4d00fd063" - - url: "https://zenodo.org/record/4472664/files/limbcoef_atmssea_noaa20.txt" - known_hash: "6853d0536b11c31dc130ab12c61fa322a76d3823a4b8ff9a18a0ecedbf269a88" - - url: "https://zenodo.org/record/4472664/files/limbcoef_atmssea_snpp.txt" + - url: "https://zenodo.org/record/10357932/files/limbcoef_atmssea_noaa20.txt" + known_hash: "07cd7874ff3f069cc3d473bdd0d1d19880ef01ac8d75cb0212a3687c059557f4" + - url: "https://zenodo.org/record/10357932/files/limbcoef_atmssea_snpp.txt" known_hash: "d0f806051b80320e046bdae6a9b68616152bbf8c2dbf3667b9834459259c0d72" file_types: From 944b049aefe93d8f2fb2a0339ce34cb555cac853 Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Tue, 12 Dec 2023 09:11:42 +0000 Subject: [PATCH 053/130] Add File handler for AMV --- satpy/readers/fci_l2_nc.py | 81 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index c387326f89..3743a64480 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -401,3 +401,84 @@ def _modify_area_extent(stand_area_extent): area_extent = tuple([ll_x, ll_y, ur_x, ur_y]) return area_extent + +class FciL2NCAMVFileHandler(FciL2CommonFunctions, BaseFileHandler): + """Reader class for FCI L2 AMV products in NetCDF4 format.""" + def __init__(self, filename, filename_info, filetype_info): + """Open the NetCDF file with xarray and prepare for dataset reading.""" + super().__init__(filename, filename_info, filetype_info) + + # Use xarray's default netcdf4 engine to open the file + self.nc = xr.open_dataset( + self.filename, + decode_cf=True, + mask_and_scale=True, + chunks={ + "number_of_images": CHUNK_SIZE, + # 'number_of_height_estimates': CHUNK_SIZE, + "number_of_winds": CHUNK_SIZE + } + ) + @property + def spacecraft_name(self): + """Get spacecraft name.""" + try: + return self.nc.attrs["platform"] + except KeyError: + # TODO if the platform attribute is not valid, return a default value + logger.warning("Spacecraft name cannot be obtained from file content, use default value instead") + return "MTI1" + + @property + def sensor_name(self): + """Get instrument name.""" + try: + return self.nc.attrs["data_source"] + except KeyError: + # TODO if the data_source attribute is not valid, return a default value + logger.warning("Sensor cannot be obtained from file content, use default value instead") + return "FCI" + + def _get_global_attributes(self): + """Create a dictionary of global attributes to be added to all datasets. + + Returns: + dict: A dictionary of global attributes. + filename: name of the product file + spacecraft_name: name of the spacecraft + sensor: name of sensor + platform_name: name of the platform + + """ + attributes = { + "filename": self.filename, + "spacecraft_name": self._spacecraft_name, + "sensor": self._sensor_name, + "platform_name": self._spacecraft_name, + "channel":self.filename_info["channel"] + } + return attributes + + def get_dataset(self, dataset_id, dataset_info): + """Get dataset using the file_key in dataset_info.""" + var_key = dataset_info["file_key"] + logger.debug("Reading in file to get dataset with key %s.", var_key) + + try: + variable = self.nc[var_key] + except KeyError: + logger.warning("Could not find key %s in NetCDF file, no valid Dataset created", var_key) + return None + + # Manage the attributes of the dataset + variable.attrs.update(dataset_info) + variable.attrs.update(self._get_global_attributes()) + + return variable + + def __del__(self): + """Close the NetCDF file that may still be open.""" + try: + self.nc.close() + except AttributeError: + pass From 0fe0ae197bc6c5b6aeb4badb4fc4e935c895f2be Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Tue, 12 Dec 2023 09:34:18 +0000 Subject: [PATCH 054/130] Fix wrong naming for method --- satpy/readers/fci_l2_nc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index 3743a64480..5d15d24528 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -452,9 +452,9 @@ def _get_global_attributes(self): """ attributes = { "filename": self.filename, - "spacecraft_name": self._spacecraft_name, - "sensor": self._sensor_name, - "platform_name": self._spacecraft_name, + "spacecraft_name": self.spacecraft_name, + "sensor": self.sensor_name, + "platform_name": self.spacecraft_name, "channel":self.filename_info["channel"] } return attributes From 9ca6d246d9e243a79354596004a8e3a44ad47add Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Tue, 12 Dec 2023 09:34:48 +0000 Subject: [PATCH 055/130] Fix reference to AMV file handler --- satpy/etc/readers/fci_l2_nc.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/etc/readers/fci_l2_nc.yaml b/satpy/etc/readers/fci_l2_nc.yaml index 7c0724b6ac..f9c12849eb 100644 --- a/satpy/etc/readers/fci_l2_nc.yaml +++ b/satpy/etc/readers/fci_l2_nc.yaml @@ -65,12 +65,12 @@ file_types: - '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+FCI-2-ASR-{subtype}-{coverage}-{subsetting}-{component1}-{component2}-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' nc_fci_amvi: - file_reader: !!python/name:readers.fci_amv_l2_nc.FciAmvL2NCFileHandler + file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCAMVFileHandler file_patterns: - '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+FCI-2-AMVI-{channel}-{coverage}-{subsetting}-{component1}-{component2}-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' nc_fci_amv: - file_reader: !!python/name:readers.fci_amv_l2_nc.FciAmvL2NCFileHandler + file_reader: !!python/name:satpy.readers.fci_l2_nc.FciL2NCAMVFileHandler file_patterns: - '{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+FCI-2-AMV-{channel}-{coverage}-{subsetting}-{component1}-{component2}-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' From f2b3238574bbf07906866d5c0be9c096731e0ccc Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Tue, 12 Dec 2023 09:43:42 +0000 Subject: [PATCH 056/130] Remove duplicate method del --- satpy/readers/fci_l2_nc.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index 5d15d24528..0948cd0e0a 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -475,10 +475,3 @@ def get_dataset(self, dataset_id, dataset_info): variable.attrs.update(self._get_global_attributes()) return variable - - def __del__(self): - """Close the NetCDF file that may still be open.""" - try: - self.nc.close() - except AttributeError: - pass From 0d2312a617fee7a9c6264c8f586700f35e569329 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 12 Dec 2023 12:49:38 +0200 Subject: [PATCH 057/130] Use create_gradient_search_resampler() --- satpy/resample.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/resample.py b/satpy/resample.py index ddab90be82..f74b6c5ecd 100644 --- a/satpy/resample.py +++ b/satpy/resample.py @@ -154,7 +154,7 @@ from packaging import version from pyresample.ewa import DaskEWAResampler, LegacyDaskEWAResampler from pyresample.geometry import SwathDefinition -from pyresample.gradient import GradientSearchResampler +from pyresample.gradient import create_gradient_search_resampler from pyresample.resampler import BaseResampler as PRBaseResampler from satpy._config import config_search_paths, get_config_path @@ -1009,7 +1009,7 @@ def compute(self, data, fill_value=np.nan, categories=None, **kwargs): "nearest": KDTreeResampler, "bilinear": BilinearResampler, "native": NativeResampler, - "gradient_search": GradientSearchResampler, + "gradient_search": create_gradient_search_resampler, "bucket_avg": BucketAvg, "bucket_sum": BucketSum, "bucket_count": BucketCount, From 7c54a14e0d1d9f40a887de88f0204e2a80305441 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 12 Dec 2023 12:50:21 +0200 Subject: [PATCH 058/130] Do not use proj dicts --- satpy/tests/test_resample.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/satpy/tests/test_resample.py b/satpy/tests/test_resample.py index 7135661578..9a0584e301 100644 --- a/satpy/tests/test_resample.py +++ b/satpy/tests/test_resample.py @@ -48,7 +48,6 @@ def get_test_data(input_shape=(100, 50), output_shape=(200, 100), output_proj=No """ import dask.array as da from pyresample.geometry import AreaDefinition, SwathDefinition - from pyresample.utils import proj4_str_to_dict from xarray import DataArray ds1 = DataArray(da.zeros(input_shape, chunks=85), dims=input_dims, @@ -62,16 +61,16 @@ def get_test_data(input_shape=(100, 50), output_shape=(200, 100), output_proj=No input_proj_str = ("+proj=geos +lon_0=-95.0 +h=35786023.0 +a=6378137.0 " "+b=6356752.31414 +sweep=x +units=m +no_defs") + crs = CRS(input_proj_str) source = AreaDefinition( "test_target", "test_target", "test_target", - proj4_str_to_dict(input_proj_str), + crs, input_shape[1], # width input_shape[0], # height (-1000., -1500., 1000., 1500.)) ds1.attrs["area"] = source - crs = CRS.from_string(input_proj_str) ds1 = ds1.assign_coords(crs=crs) ds2 = ds1.copy() @@ -95,7 +94,7 @@ def get_test_data(input_shape=(100, 50), output_shape=(200, 100), output_proj=No "test_target", "test_target", "test_target", - proj4_str_to_dict(output_proj_str), + CRS(output_proj_str), output_shape[1], # width output_shape[0], # height (-1000., -1500., 1000., 1500.), From c72cee28dacd0a1ab2d4e49843f8180885f87a50 Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Tue, 12 Dec 2023 10:54:29 +0000 Subject: [PATCH 059/130] Add test for AMV reader --- satpy/tests/reader_tests/test_fci_l2_nc.py | 81 +++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index 22611a8469..fb5c725ffc 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -29,7 +29,7 @@ from netCDF4 import Dataset from pyresample import geometry -from satpy.readers.fci_l2_nc import FciL2NCFileHandler, FciL2NCSegmentFileHandler +from satpy.readers.fci_l2_nc import FciL2NCAMVFileHandler, FciL2NCFileHandler, FciL2NCSegmentFileHandler from satpy.tests.utils import make_dataid AREA_DEF = geometry.AreaDefinition( @@ -507,3 +507,82 @@ def test_byte_extraction(self): }) assert dataset.values == 0 + +class TestFciL2NCAMVFileHandler(unittest.TestCase): + """Test the FciL2NCFileHandler reader.""" + + def setUp(self): + """Set up the test by creating a test file and opening it with the reader.""" + # Easiest way to test the reader is to create a test netCDF file on the fly + # Create unique filenames to prevent race conditions when tests are run in parallel + self.test_file = str(uuid.uuid4()) + ".nc" + with Dataset(self.test_file, "w") as nc: + # Create dimensions + nc.createDimension("number_of_winds", 50000) + + # add global attributes + nc.data_source = "test_data_source" + nc.platform = "test_platform" + + # Add datasets + latitude = nc.createVariable("latitude", np.float32, dimensions=("number_of_winds",)) + latitude[:] = np.arange(50000) + + longitude = nc.createVariable("y", np.float32, dimensions=("number_of_winds",)) + longitude[:] = np.arange(50000) + + qi = nc.createVariable("product_quality", np.int8) + qi[:] = 99. + + test_dataset = nc.createVariable("test_one_layer", np.float32, + dimensions="number_of_winds") + test_dataset[:] = np.ones((50000)) + test_dataset.test_attr = "attr" + test_dataset.units = "test_units" + + mtg_geos_projection = nc.createVariable("mtg_geos_projection", int, dimensions=()) + mtg_geos_projection.longitude_of_projection_origin = 0.0 + mtg_geos_projection.semi_major_axis = 6378137. + mtg_geos_projection.inverse_flattening = 298.257223563 + mtg_geos_projection.perspective_point_height = 35786400. + + self.fh = FciL2NCAMVFileHandler(filename=self.test_file, + filename_info={"channel":"test_channel"}, + filetype_info={}) + + def tearDown(self): + """Remove the previously created test file.""" + # First delete the file handler, forcing the file to be closed if still open + del self.fh + # Then we can safely remove the file from the system + with suppress(OSError): + os.remove(self.test_file) + + def test_all_basic(self): + """Test all basic functionalities.""" + assert self.fh.spacecraft_name == "test_platform" + assert self.fh.sensor_name == "test_data_source" + assert self.fh.ssp_lon == 0.0 + + global_attributes = self.fh._get_global_attributes() + expected_global_attributes = { + "filename": self.test_file, + "spacecraft_name": "test_platform", + "sensor": "test_data_source", + "platform_name": "test_platform", + "channel": "test_channel" + } + assert global_attributes == expected_global_attributes + + def test_dataset(self): + """Test the correct execution of the get_dataset function with a valid file_key.""" + dataset = self.fh.get_dataset(make_dataid(name="test_dataset", resolution=2000), + {"name": "test_dataset", + "file_key": "test_dataset", + "fill_value": -999, + "file_type": "test_file_type"}) + + np.testing.assert_allclose(dataset.values, np.ones((50000))) + assert dataset.attrs["test_attr"] == "attr" + assert dataset.attrs["units"] == "test_units" + assert dataset.attrs["fill_value"] == -999 From a7f93eb9eecf5919bb42071251c3092864a7589b Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 12 Dec 2023 13:12:34 +0200 Subject: [PATCH 060/130] Remove unnecessary (since Pyresample 1.18) mask_all_nan/skipna handling --- satpy/resample.py | 47 +++---------------------- satpy/tests/test_resample.py | 68 ------------------------------------ 2 files changed, 4 insertions(+), 111 deletions(-) diff --git a/satpy/resample.py b/satpy/resample.py index f74b6c5ecd..336e3fec11 100644 --- a/satpy/resample.py +++ b/satpy/resample.py @@ -148,10 +148,8 @@ import dask.array as da import numpy as np -import pyresample import xarray as xr import zarr -from packaging import version from pyresample.ewa import DaskEWAResampler, LegacyDaskEWAResampler from pyresample.geometry import SwathDefinition from pyresample.gradient import create_gradient_search_resampler @@ -177,8 +175,6 @@ resamplers_cache: "WeakValueDictionary[tuple, object]" = WeakValueDictionary() -PR_USE_SKIPNA = version.parse(pyresample.__version__) > version.parse("1.17.0") - def hash_dict(the_dict, the_hash=None): """Calculate a hash for a dictionary.""" @@ -773,33 +769,6 @@ def _get_replicated_chunk_sizes(d_arr, repeats): return tuple(repeated_chunks) -def _get_arg_to_pass_for_skipna_handling(**kwargs): - """Determine if skipna can be passed to the compute functions for the average and sum bucket resampler.""" - # FIXME this can be removed once Pyresample 1.18.0 is a Satpy requirement - - if PR_USE_SKIPNA: - if "mask_all_nan" in kwargs: - warnings.warn( - "Argument mask_all_nan is deprecated. Please use skipna for missing values handling. " - "Continuing with default skipna=True, if not provided differently.", - DeprecationWarning, - stacklevel=3 - ) - kwargs.pop("mask_all_nan") - else: - if "mask_all_nan" in kwargs: - warnings.warn( - "Argument mask_all_nan is deprecated." - "Please update Pyresample and use skipna for missing values handling.", - DeprecationWarning, - stacklevel=3 - ) - kwargs.setdefault("mask_all_nan", False) - kwargs.pop("skipna") - - return kwargs - - class BucketResamplerBase(PRBaseResampler): """Base class for bucket resampling which implements averaging.""" @@ -832,11 +801,6 @@ def resample(self, data, **kwargs): # noqa: D417 Returns (xarray.DataArray): Data resampled to the target area """ - if not PR_USE_SKIPNA and "skipna" in kwargs: - raise ValueError("You are trying to set the skipna argument but you are using an old version of" - " Pyresample that does not support it." - "Please update Pyresample to 1.18.0 or higher to be able to use this argument.") - self.precompute(**kwargs) attrs = data.attrs.copy() data_arr = data.data @@ -910,17 +874,16 @@ def compute(self, data, fill_value=np.nan, skipna=True, **kwargs): # noqa: D417 Returns: dask.Array """ - kwargs = _get_arg_to_pass_for_skipna_handling(skipna=skipna, **kwargs) - results = [] if data.ndim == 3: for i in range(data.shape[0]): res = self.resampler.get_average(data[i, :, :], fill_value=fill_value, + skipna=skipna, **kwargs) results.append(res) else: - res = self.resampler.get_average(data, fill_value=fill_value, + res = self.resampler.get_average(data, fill_value=fill_value, skipna=skipna, **kwargs) results.append(res) @@ -948,16 +911,14 @@ class BucketSum(BucketResamplerBase): def compute(self, data, skipna=True, **kwargs): """Call the resampling.""" - kwargs = _get_arg_to_pass_for_skipna_handling(skipna=skipna, **kwargs) - results = [] if data.ndim == 3: for i in range(data.shape[0]): - res = self.resampler.get_sum(data[i, :, :], + res = self.resampler.get_sum(data[i, :, :], skipna=skipna, **kwargs) results.append(res) else: - res = self.resampler.get_sum(data, **kwargs) + res = self.resampler.get_sum(data, skipna=skipna, **kwargs) results.append(res) return da.stack(results) diff --git a/satpy/tests/test_resample.py b/satpy/tests/test_resample.py index 9a0584e301..d0bbbe2a46 100644 --- a/satpy/tests/test_resample.py +++ b/satpy/tests/test_resample.py @@ -581,17 +581,10 @@ def test_compute(self): res = self._compute_mocked_bucket_avg(data, return_data=data[0, :, :], fill_value=2) assert res.shape == (3, 5, 5) - @mock.patch("satpy.resample.PR_USE_SKIPNA", True) def test_compute_and_use_skipna_handling(self): """Test bucket resampler computation and use skipna handling.""" data = da.ones((5,)) - self._compute_mocked_bucket_avg(data, fill_value=2, mask_all_nan=True) - self.bucket.resampler.get_average.assert_called_once_with( - data, - fill_value=2, - skipna=True) - self._compute_mocked_bucket_avg(data, fill_value=2, skipna=False) self.bucket.resampler.get_average.assert_called_once_with( data, @@ -604,35 +597,6 @@ def test_compute_and_use_skipna_handling(self): fill_value=2, skipna=True) - @mock.patch("satpy.resample.PR_USE_SKIPNA", False) - def test_compute_and_not_use_skipna_handling(self): - """Test bucket resampler computation and not use skipna handling.""" - data = da.ones((5,)) - - self._compute_mocked_bucket_avg(data, fill_value=2, mask_all_nan=True) - self.bucket.resampler.get_average.assert_called_once_with( - data, - fill_value=2, - mask_all_nan=True) - - self._compute_mocked_bucket_avg(data, fill_value=2, mask_all_nan=False) - self.bucket.resampler.get_average.assert_called_once_with( - data, - fill_value=2, - mask_all_nan=False) - - self._compute_mocked_bucket_avg(data, fill_value=2) - self.bucket.resampler.get_average.assert_called_once_with( - data, - fill_value=2, - mask_all_nan=False) - - self._compute_mocked_bucket_avg(data, fill_value=2, skipna=True) - self.bucket.resampler.get_average.assert_called_once_with( - data, - fill_value=2, - mask_all_nan=False) - @mock.patch("pyresample.bucket.BucketResampler") def test_resample(self, pyresample_bucket): """Test bucket resamplers resample method.""" @@ -712,16 +676,10 @@ def test_compute(self): res = self._compute_mocked_bucket_sum(data, return_data=data[0, :, :]) assert res.shape == (3, 5, 5) - @mock.patch("satpy.resample.PR_USE_SKIPNA", True) def test_compute_and_use_skipna_handling(self): """Test bucket resampler computation and use skipna handling.""" data = da.ones((5,)) - self._compute_mocked_bucket_sum(data, mask_all_nan=True) - self.bucket.resampler.get_sum.assert_called_once_with( - data, - skipna=True) - self._compute_mocked_bucket_sum(data, skipna=False) self.bucket.resampler.get_sum.assert_called_once_with( data, @@ -732,32 +690,6 @@ def test_compute_and_use_skipna_handling(self): data, skipna=True) - @mock.patch("satpy.resample.PR_USE_SKIPNA", False) - def test_compute_and_not_use_skipna_handling(self): - """Test bucket resampler computation and not use skipna handling.""" - data = da.ones((5,)) - - self._compute_mocked_bucket_sum(data, mask_all_nan=True) - self.bucket.resampler.get_sum.assert_called_once_with( - data, - mask_all_nan=True) - - self._compute_mocked_bucket_sum(data, mask_all_nan=False) - self.bucket.resampler.get_sum.assert_called_once_with( - data, - mask_all_nan=False) - - self._compute_mocked_bucket_sum(data) - self.bucket.resampler.get_sum.assert_called_once_with( - data, - mask_all_nan=False) - - self._compute_mocked_bucket_sum(data, fill_value=2, skipna=True) - self.bucket.resampler.get_sum.assert_called_once_with( - data, - fill_value=2, - mask_all_nan=False) - class TestBucketCount(unittest.TestCase): """Test the count bucket resampler.""" From 5f00f09efe3412b06b32d481a6dca26c56e42e5d Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 12 Dec 2023 13:17:32 +0200 Subject: [PATCH 061/130] Catch re-chunking warning --- satpy/tests/test_resample.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/satpy/tests/test_resample.py b/satpy/tests/test_resample.py index d0bbbe2a46..11a9644eb4 100644 --- a/satpy/tests/test_resample.py +++ b/satpy/tests/test_resample.py @@ -247,8 +247,12 @@ def test_expand_reduce_agg_rechunk(self): into that chunk size. """ + from satpy.utils import PerformanceWarning + d_arr = da.zeros((6, 20), chunks=3) - new_data = NativeResampler._expand_reduce(d_arr, {0: 0.5, 1: 0.5}) + text = "Array chunk size is not divisible by aggregation factor. Re-chunking to continue native resampling." + with pytest.warns(PerformanceWarning, match=text): + new_data = NativeResampler._expand_reduce(d_arr, {0: 0.5, 1: 0.5}) assert new_data.shape == (3, 10) def test_expand_reduce_numpy(self): From 084d5031d279c93216d740dad1509182767f44b7 Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Tue, 12 Dec 2023 11:39:07 +0000 Subject: [PATCH 062/130] Update AHI HSD reader to correctly handle singleton arrays. --- satpy/readers/ahi_hsd.py | 30 ++++++++++++------------ satpy/tests/reader_tests/test_ahi_hsd.py | 12 +++++----- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/satpy/readers/ahi_hsd.py b/satpy/readers/ahi_hsd.py index 8e14d049b9..889b858ff5 100644 --- a/satpy/readers/ahi_hsd.py +++ b/satpy/readers/ahi_hsd.py @@ -419,12 +419,12 @@ def end_time(self): @property def observation_start_time(self): """Get the observation start time.""" - return datetime(1858, 11, 17) + timedelta(days=float(self.basic_info["observation_start_time"])) + return datetime(1858, 11, 17) + timedelta(days=float(self.basic_info["observation_start_time"][0])) @property def observation_end_time(self): """Get the observation end time.""" - return datetime(1858, 11, 17) + timedelta(days=float(self.basic_info["observation_end_time"])) + return datetime(1858, 11, 17) + timedelta(days=float(self.basic_info["observation_end_time"][0])) @property def nominal_start_time(self): @@ -498,8 +498,8 @@ def _get_area_def(self): pdict["h"] = float(self.proj_info["distance_from_earth_center"] * 1000 - pdict["a"]) pdict["b"] = float(self.proj_info["earth_polar_radius"] * 1000) pdict["ssp_lon"] = float(self.proj_info["sub_lon"]) - pdict["nlines"] = int(self.data_info["number_of_lines"]) - pdict["ncols"] = int(self.data_info["number_of_columns"]) + pdict["nlines"] = int(self.data_info["number_of_lines"][0]) + pdict["ncols"] = int(self.data_info["number_of_columns"][0]) pdict["scandir"] = "N2S" pdict["loff"] = pdict["loff"] + (self.segment_number * pdict["nlines"]) @@ -528,19 +528,19 @@ def _read_header(self, fp_): fpos = 0 header["block1"] = np.fromfile( fp_, dtype=_BASIC_INFO_TYPE, count=1) - fpos = fpos + int(header["block1"]["blocklength"]) + fpos = fpos + int(header["block1"]["blocklength"][0]) self._check_fpos(fp_, fpos, 0, "block1") fp_.seek(fpos, 0) header["block2"] = np.fromfile(fp_, dtype=_DATA_INFO_TYPE, count=1) - fpos = fpos + int(header["block2"]["blocklength"]) + fpos = fpos + int(header["block2"]["blocklength"][0]) self._check_fpos(fp_, fpos, 0, "block2") fp_.seek(fpos, 0) header["block3"] = np.fromfile(fp_, dtype=_PROJ_INFO_TYPE, count=1) - fpos = fpos + int(header["block3"]["blocklength"]) + fpos = fpos + int(header["block3"]["blocklength"][0]) self._check_fpos(fp_, fpos, 0, "block3") fp_.seek(fpos, 0) header["block4"] = np.fromfile(fp_, dtype=_NAV_INFO_TYPE, count=1) - fpos = fpos + int(header["block4"]["blocklength"]) + fpos = fpos + int(header["block4"]["blocklength"][0]) self._check_fpos(fp_, fpos, 0, "block4") fp_.seek(fpos, 0) header["block5"] = np.fromfile(fp_, dtype=_CAL_INFO_TYPE, count=1) @@ -553,7 +553,7 @@ def _read_header(self, fp_): cal = np.fromfile(fp_, dtype=_VISCAL_INFO_TYPE, count=1) else: cal = np.fromfile(fp_, dtype=_IRCAL_INFO_TYPE, count=1) - fpos = fpos + int(header["block5"]["blocklength"]) + fpos = fpos + int(header["block5"]["blocklength"][0]) self._check_fpos(fp_, fpos, 0, "block5") fp_.seek(fpos, 0) @@ -561,12 +561,12 @@ def _read_header(self, fp_): header["block6"] = np.fromfile( fp_, dtype=_INTER_CALIBRATION_INFO_TYPE, count=1) - fpos = fpos + int(header["block6"]["blocklength"]) + fpos = fpos + int(header["block6"]["blocklength"][0]) self._check_fpos(fp_, fpos, 0, "block6") fp_.seek(fpos, 0) header["block7"] = np.fromfile( fp_, dtype=_SEGMENT_INFO_TYPE, count=1) - fpos = fpos + int(header["block7"]["blocklength"]) + fpos = fpos + int(header["block7"]["blocklength"][0]) self._check_fpos(fp_, fpos, 0, "block7") fp_.seek(fpos, 0) header["block8"] = np.fromfile( @@ -576,7 +576,7 @@ def _read_header(self, fp_): corrections = [] for _i in range(ncorrs): corrections.append(np.fromfile(fp_, dtype=_NAVIGATION_CORRECTION_SUBINFO_TYPE, count=1)) - fpos = fpos + int(header["block8"]["blocklength"]) + fpos = fpos + int(header["block8"]["blocklength"][0]) self._check_fpos(fp_, fpos, 40, "block8") fp_.seek(fpos, 0) header["navigation_corrections"] = corrections @@ -591,7 +591,7 @@ def _read_header(self, fp_): dtype=_OBSERVATION_LINE_TIME_INFO_TYPE, count=1)) header["observation_time_information"] = lines_and_times - fpos = fpos + int(header["block9"]["blocklength"]) + fpos = fpos + int(header["block9"]["blocklength"][0]) self._check_fpos(fp_, fpos, 40, "block9") fp_.seek(fpos, 0) @@ -604,12 +604,12 @@ def _read_header(self, fp_): for _i in range(num_err_info_data): err_info_data.append(np.fromfile(fp_, dtype=_ERROR_LINE_INFO_TYPE, count=1)) header["error_information_data"] = err_info_data - fpos = fpos + int(header["block10"]["blocklength"]) + fpos = fpos + int(header["block10"]["blocklength"][0]) self._check_fpos(fp_, fpos, 40, "block10") fp_.seek(fpos, 0) header["block11"] = np.fromfile(fp_, dtype=_SPARE_TYPE, count=1) - fpos = fpos + int(header["block11"]["blocklength"]) + fpos = fpos + int(header["block11"]["blocklength"][0]) self._check_fpos(fp_, fpos, 0, "block11") fp_.seek(fpos, 0) diff --git a/satpy/tests/reader_tests/test_ahi_hsd.py b/satpy/tests/reader_tests/test_ahi_hsd.py index 9338440246..2075b88947 100644 --- a/satpy/tests/reader_tests/test_ahi_hsd.py +++ b/satpy/tests/reader_tests/test_ahi_hsd.py @@ -48,8 +48,8 @@ "compression_flag_for_data": 0, "hblock_number": 2, "number_of_bits_per_pixel": 16, - "number_of_columns": 11000, - "number_of_lines": 1100, + "number_of_columns": [11000], + "number_of_lines": [1100], "spare": "", } FAKE_PROJ_INFO: InfoDict = { @@ -135,8 +135,8 @@ def test_region(self, fromfile, np2str): "compression_flag_for_data": 0, "hblock_number": 2, "number_of_bits_per_pixel": 16, - "number_of_columns": 1000, - "number_of_lines": 1000, + "number_of_columns": [1000], + "number_of_lines": [1000], "spare": ""} area_def = fh.get_area_def(None) @@ -183,8 +183,8 @@ def test_segment(self, fromfile, np2str): "compression_flag_for_data": 0, "hblock_number": 2, "number_of_bits_per_pixel": 16, - "number_of_columns": 11000, - "number_of_lines": 1100, + "number_of_columns": [11000], + "number_of_lines": [1100], "spare": ""} area_def = fh.get_area_def(None) From b00a6d93bb27283f440fb593154f612ec2fe4a9f Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Tue, 12 Dec 2023 12:06:46 +0000 Subject: [PATCH 063/130] Fix test for get_dataset --- satpy/tests/reader_tests/test_fci_l2_nc.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index fb5c725ffc..8eff51e344 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -534,9 +534,9 @@ def setUp(self): qi = nc.createVariable("product_quality", np.int8) qi[:] = 99. - test_dataset = nc.createVariable("test_one_layer", np.float32, + test_dataset = nc.createVariable("test_dataset", np.float32, dimensions="number_of_winds") - test_dataset[:] = np.ones((50000)) + test_dataset[:] = np.ones(50000) test_dataset.test_attr = "attr" test_dataset.units = "test_units" @@ -581,8 +581,7 @@ def test_dataset(self): "file_key": "test_dataset", "fill_value": -999, "file_type": "test_file_type"}) - - np.testing.assert_allclose(dataset.values, np.ones((50000))) + np.testing.assert_allclose(dataset.values, np.ones(50000)) assert dataset.attrs["test_attr"] == "attr" assert dataset.attrs["units"] == "test_units" assert dataset.attrs["fill_value"] == -999 From 610d365c247393fe50b7dfce7a675eeab634bd22 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 12 Dec 2023 14:07:50 +0200 Subject: [PATCH 064/130] Suppress division-by-zero warning in RatioSharpenedRGB --- satpy/composites/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 9295f94dc7..71b9bd0605 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1180,7 +1180,8 @@ def _combined_sharpened_info(self, info, new_attrs): def _get_sharpening_ratio(high_res, low_res): - ratio = high_res / low_res + with np.errstate(divide="ignore"): + ratio = high_res / low_res # make ratio a no-op (multiply by 1) where the ratio is NaN, infinity, # or it is negative. ratio[~np.isfinite(ratio) | (ratio < 0)] = 1.0 From 51ab6b7f00ca5e9e6073f55cc8529051db881922 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 12 Dec 2023 14:33:28 +0200 Subject: [PATCH 065/130] Use ds.drop_vars(), adjust XArray version requirement to match --- satpy/composites/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 71b9bd0605..17d4c00075 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -204,7 +204,7 @@ def drop_coordinates(self, data_arrays): if coord not in ds.dims and any([neglible in coord for neglible in NEGLIGIBLE_COORDS])] if drop: - new_arrays.append(ds.drop(drop)) + new_arrays.append(ds.drop_vars(drop)) else: new_arrays.append(ds) diff --git a/setup.py b/setup.py index cd1c43422e..a9bf050786 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ from setuptools import find_packages, setup requires = ["numpy >=1.21", "pillow", "pyresample >=1.24.0", "trollsift", - "trollimage >=1.20", "pykdtree", "pyyaml >=5.1", "xarray >=0.10.1, !=0.13.0", + "trollimage >=1.20", "pykdtree", "pyyaml >=5.1", "xarray >=0.14.1", "dask[array] >=0.17.1", "pyproj>=2.2", "zarr", "donfig", "appdirs", "packaging", "pooch", "pyorbital"] From 5c11a5be684a517d4dada08e993708b4aadc89bf Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 12 Dec 2023 14:46:26 +0200 Subject: [PATCH 066/130] Remove deprecated GreenCorrector --- satpy/composites/spectral.py | 21 ------------------- satpy/tests/compositor_tests/test_spectral.py | 14 +------------ 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/satpy/composites/spectral.py b/satpy/composites/spectral.py index 448d7cb26a..d656bab7ec 100644 --- a/satpy/composites/spectral.py +++ b/satpy/composites/spectral.py @@ -16,7 +16,6 @@ """Composite classes for spectral adjustments.""" import logging -import warnings from satpy.composites import GenericCompositor from satpy.dataset import combine_metadata @@ -199,23 +198,3 @@ def _compute_blend_fraction(self, ndvi): + self.limits[0] return fraction - - -class GreenCorrector(SpectralBlender): - """Previous class used to blend channels for green band corrections. - - This method has been refactored to make it more generic. The replacement class is 'SpectralBlender' which computes - a weighted average based on N number of channels and N number of corresponding weights/fractions. A new class - called 'HybridGreen' has been created, which performs a correction of green bands centered at 0.51 microns - following Miller et al. (2016, :doi:`10.1175/BAMS-D-15-00154.2`) in order to improve true color imagery. - """ - - def __init__(self, *args, fractions=(0.85, 0.15), **kwargs): - """Set default keyword argument values.""" - warnings.warn( - "'GreenCorrector' is deprecated, use 'SpectralBlender' instead, or 'HybridGreen' for hybrid green" - " correction following Miller et al. (2016).", - UserWarning, - stacklevel=2 - ) - super().__init__(fractions=fractions, *args, **kwargs) diff --git a/satpy/tests/compositor_tests/test_spectral.py b/satpy/tests/compositor_tests/test_spectral.py index e46cff4d0c..c7f07c0454 100644 --- a/satpy/tests/compositor_tests/test_spectral.py +++ b/satpy/tests/compositor_tests/test_spectral.py @@ -21,7 +21,7 @@ import pytest import xarray as xr -from satpy.composites.spectral import GreenCorrector, HybridGreen, NDVIHybridGreen, SpectralBlender +from satpy.composites.spectral import HybridGreen, NDVIHybridGreen, SpectralBlender from satpy.tests.utils import CustomScheduler @@ -67,18 +67,6 @@ def test_hybrid_green(self): data = res.compute() np.testing.assert_allclose(data, 0.23) - def test_green_corrector(self): - """Test the deprecated class for green corrections.""" - comp = GreenCorrector("blended_channel", fractions=(0.85, 0.15), prerequisites=(0.51, 0.85), - standard_name="toa_bidirectional_reflectance") - res = comp((self.c01, self.c03)) - assert isinstance(res, xr.DataArray) - assert isinstance(res.data, da.Array) - assert res.attrs["name"] == "blended_channel" - assert res.attrs["standard_name"] == "toa_bidirectional_reflectance" - data = res.compute() - np.testing.assert_allclose(data, 0.23) - class TestNdviHybridGreenCompositor: """Test NDVI-weighted hybrid green correction of green band.""" From 934bdb30945189d5d9e65757767ef860c3b91bfb Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Tue, 12 Dec 2023 13:04:55 +0000 Subject: [PATCH 067/130] Remove duplicate methods --- satpy/readers/fci_l2_nc.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index 0948cd0e0a..03f8e94f55 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -419,25 +419,6 @@ def __init__(self, filename, filename_info, filetype_info): "number_of_winds": CHUNK_SIZE } ) - @property - def spacecraft_name(self): - """Get spacecraft name.""" - try: - return self.nc.attrs["platform"] - except KeyError: - # TODO if the platform attribute is not valid, return a default value - logger.warning("Spacecraft name cannot be obtained from file content, use default value instead") - return "MTI1" - - @property - def sensor_name(self): - """Get instrument name.""" - try: - return self.nc.attrs["data_source"] - except KeyError: - # TODO if the data_source attribute is not valid, return a default value - logger.warning("Sensor cannot be obtained from file content, use default value instead") - return "FCI" def _get_global_attributes(self): """Create a dictionary of global attributes to be added to all datasets. From 3d6d561986b433264f918aed3646387ce9fa676a Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Tue, 12 Dec 2023 13:07:40 +0000 Subject: [PATCH 068/130] Add test for invalide dataset --- satpy/tests/reader_tests/test_fci_l2_nc.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index 8eff51e344..7853cba900 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -585,3 +585,12 @@ def test_dataset(self): assert dataset.attrs["test_attr"] == "attr" assert dataset.attrs["units"] == "test_units" assert dataset.attrs["fill_value"] == -999 + + def test_dataset_with_invalid_filekey(self): + """Test the correct execution of the get_dataset function with an invalid file_key.""" + invalid_dataset = self.fh.get_dataset(make_dataid(name="test_invalid", resolution=2000), + {"name": "test_invalid", + "file_key": "test_invalid", + "fill_value": -999, + "file_type": "test_file_type"}) + assert invalid_dataset is None From bb56a486200b6436d00b9922f670ec5d0f40c3d0 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 12 Dec 2023 15:11:20 +0200 Subject: [PATCH 069/130] Remove GreenCorrector import --- satpy/composites/ahi.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/satpy/composites/ahi.py b/satpy/composites/ahi.py index bb96a94581..4826f84820 100644 --- a/satpy/composites/ahi.py +++ b/satpy/composites/ahi.py @@ -14,7 +14,3 @@ # You should have received a copy of the GNU General Public License along with # satpy. If not, see . """Composite classes for AHI.""" - -# The green corrector used to be defined here, but was moved to spectral.py -# in Satpy 0.38 because it also applies to FCI. -from .spectral import GreenCorrector # noqa: F401 From f60b188938d16c0d365eaec8d4c961f8ec0a1a62 Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Tue, 12 Dec 2023 13:16:53 +0000 Subject: [PATCH 070/130] Use `item` to select singleton array elements in AHI HSD. --- satpy/readers/ahi_hsd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/readers/ahi_hsd.py b/satpy/readers/ahi_hsd.py index 889b858ff5..cf3fe018f5 100644 --- a/satpy/readers/ahi_hsd.py +++ b/satpy/readers/ahi_hsd.py @@ -419,12 +419,12 @@ def end_time(self): @property def observation_start_time(self): """Get the observation start time.""" - return datetime(1858, 11, 17) + timedelta(days=float(self.basic_info["observation_start_time"][0])) + return datetime(1858, 11, 17) + timedelta(days=float(self.basic_info["observation_start_time"].item())) @property def observation_end_time(self): """Get the observation end time.""" - return datetime(1858, 11, 17) + timedelta(days=float(self.basic_info["observation_end_time"][0])) + return datetime(1858, 11, 17) + timedelta(days=float(self.basic_info["observation_end_time"].item())) @property def nominal_start_time(self): From 336fc9c97a08d9f367495ae356a7f0d6dcb4d0dd Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 12 Dec 2023 15:19:30 +0200 Subject: [PATCH 071/130] Remove deprecated AHI composites --- satpy/etc/composites/ahi.yaml | 40 ----------------------------------- 1 file changed, 40 deletions(-) diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index cda79a5fac..9c585d53de 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -15,46 +15,6 @@ modifiers: - solar_zenith_angle composites: - green: - deprecation_warning: "'green' is a deprecated composite. Use the equivalent 'hybrid_green' instead." - compositor: !!python/name:satpy.composites.spectral.HybridGreen - # FUTURE: Set a wavelength...see what happens. Dependency finding - # probably wouldn't work. - prerequisites: - # should we be using the most corrected or least corrected inputs? - # what happens if something requests more modifiers on top of this? - - wavelength: 0.51 - modifiers: [sunz_corrected, rayleigh_corrected] - - wavelength: 0.85 - modifiers: [sunz_corrected] - standard_name: toa_bidirectional_reflectance - - green_true_color_reproduction: - # JMA True Color Reproduction green band - # http://www.jma.go.jp/jma/jma-eng/satellite/introduction/TCR.html - deprecation_warning: "'green_true_color_reproduction' is a deprecated composite. Use the equivalent 'reproduced_green' instead." - compositor: !!python/name:satpy.composites.spectral.SpectralBlender - fractions: [0.6321, 0.2928, 0.0751] - prerequisites: - - name: B02 - modifiers: [sunz_corrected, rayleigh_corrected] - - name: B03 - modifiers: [sunz_corrected, rayleigh_corrected] - - name: B04 - modifiers: [sunz_corrected] - standard_name: none - - green_nocorr: - deprecation_warning: "'green_nocorr' is a deprecated composite. Use the equivalent 'hybrid_green_nocorr' instead." - compositor: !!python/name:satpy.composites.spectral.HybridGreen - # FUTURE: Set a wavelength...see what happens. Dependency finding - # probably wouldn't work. - prerequisites: - # should we be using the most corrected or least corrected inputs? - # what happens if something requests more modifiers on top of this? - - wavelength: 0.51 - - wavelength: 0.85 - standard_name: toa_reflectance hybrid_green: compositor: !!python/name:satpy.composites.spectral.HybridGreen From 2469168fba6d8f5557f01bafc5ee0c67a3a73bb5 Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Tue, 12 Dec 2023 14:26:37 +0000 Subject: [PATCH 072/130] Use `item` to select singleton array elements in AHI HSD. --- satpy/readers/ahi_hsd.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/satpy/readers/ahi_hsd.py b/satpy/readers/ahi_hsd.py index cf3fe018f5..313e5ccab5 100644 --- a/satpy/readers/ahi_hsd.py +++ b/satpy/readers/ahi_hsd.py @@ -498,8 +498,8 @@ def _get_area_def(self): pdict["h"] = float(self.proj_info["distance_from_earth_center"] * 1000 - pdict["a"]) pdict["b"] = float(self.proj_info["earth_polar_radius"] * 1000) pdict["ssp_lon"] = float(self.proj_info["sub_lon"]) - pdict["nlines"] = int(self.data_info["number_of_lines"][0]) - pdict["ncols"] = int(self.data_info["number_of_columns"][0]) + pdict["nlines"] = int(self.data_info["number_of_lines"].item()) + pdict["ncols"] = int(self.data_info["number_of_columns"].item()) pdict["scandir"] = "N2S" pdict["loff"] = pdict["loff"] + (self.segment_number * pdict["nlines"]) @@ -528,19 +528,19 @@ def _read_header(self, fp_): fpos = 0 header["block1"] = np.fromfile( fp_, dtype=_BASIC_INFO_TYPE, count=1) - fpos = fpos + int(header["block1"]["blocklength"][0]) + fpos = fpos + int(header["block1"]["blocklength"].item()) self._check_fpos(fp_, fpos, 0, "block1") fp_.seek(fpos, 0) header["block2"] = np.fromfile(fp_, dtype=_DATA_INFO_TYPE, count=1) - fpos = fpos + int(header["block2"]["blocklength"][0]) + fpos = fpos + int(header["block2"]["blocklength"].item()) self._check_fpos(fp_, fpos, 0, "block2") fp_.seek(fpos, 0) header["block3"] = np.fromfile(fp_, dtype=_PROJ_INFO_TYPE, count=1) - fpos = fpos + int(header["block3"]["blocklength"][0]) + fpos = fpos + int(header["block3"]["blocklength"].item()) self._check_fpos(fp_, fpos, 0, "block3") fp_.seek(fpos, 0) header["block4"] = np.fromfile(fp_, dtype=_NAV_INFO_TYPE, count=1) - fpos = fpos + int(header["block4"]["blocklength"][0]) + fpos = fpos + int(header["block4"]["blocklength"].item()) self._check_fpos(fp_, fpos, 0, "block4") fp_.seek(fpos, 0) header["block5"] = np.fromfile(fp_, dtype=_CAL_INFO_TYPE, count=1) @@ -553,7 +553,7 @@ def _read_header(self, fp_): cal = np.fromfile(fp_, dtype=_VISCAL_INFO_TYPE, count=1) else: cal = np.fromfile(fp_, dtype=_IRCAL_INFO_TYPE, count=1) - fpos = fpos + int(header["block5"]["blocklength"][0]) + fpos = fpos + int(header["block5"]["blocklength"].item()) self._check_fpos(fp_, fpos, 0, "block5") fp_.seek(fpos, 0) @@ -561,12 +561,12 @@ def _read_header(self, fp_): header["block6"] = np.fromfile( fp_, dtype=_INTER_CALIBRATION_INFO_TYPE, count=1) - fpos = fpos + int(header["block6"]["blocklength"][0]) + fpos = fpos + int(header["block6"]["blocklength"].item()) self._check_fpos(fp_, fpos, 0, "block6") fp_.seek(fpos, 0) header["block7"] = np.fromfile( fp_, dtype=_SEGMENT_INFO_TYPE, count=1) - fpos = fpos + int(header["block7"]["blocklength"][0]) + fpos = fpos + int(header["block7"]["blocklength"].item()) self._check_fpos(fp_, fpos, 0, "block7") fp_.seek(fpos, 0) header["block8"] = np.fromfile( @@ -576,7 +576,7 @@ def _read_header(self, fp_): corrections = [] for _i in range(ncorrs): corrections.append(np.fromfile(fp_, dtype=_NAVIGATION_CORRECTION_SUBINFO_TYPE, count=1)) - fpos = fpos + int(header["block8"]["blocklength"][0]) + fpos = fpos + int(header["block8"]["blocklength"].item()) self._check_fpos(fp_, fpos, 40, "block8") fp_.seek(fpos, 0) header["navigation_corrections"] = corrections @@ -591,7 +591,7 @@ def _read_header(self, fp_): dtype=_OBSERVATION_LINE_TIME_INFO_TYPE, count=1)) header["observation_time_information"] = lines_and_times - fpos = fpos + int(header["block9"]["blocklength"][0]) + fpos = fpos + int(header["block9"]["blocklength"].item()) self._check_fpos(fp_, fpos, 40, "block9") fp_.seek(fpos, 0) @@ -604,12 +604,12 @@ def _read_header(self, fp_): for _i in range(num_err_info_data): err_info_data.append(np.fromfile(fp_, dtype=_ERROR_LINE_INFO_TYPE, count=1)) header["error_information_data"] = err_info_data - fpos = fpos + int(header["block10"]["blocklength"][0]) + fpos = fpos + int(header["block10"]["blocklength"].item()) self._check_fpos(fp_, fpos, 40, "block10") fp_.seek(fpos, 0) header["block11"] = np.fromfile(fp_, dtype=_SPARE_TYPE, count=1) - fpos = fpos + int(header["block11"]["blocklength"][0]) + fpos = fpos + int(header["block11"]["blocklength"].item()) self._check_fpos(fp_, fpos, 0, "block11") fp_.seek(fpos, 0) @@ -617,8 +617,8 @@ def _read_header(self, fp_): def _read_data(self, fp_, header, resolution): """Read data block.""" - nlines = int(header["block2"]["number_of_lines"][0]) - ncols = int(header["block2"]["number_of_columns"][0]) + nlines = int(header["block2"]["number_of_lines"].item()) + ncols = int(header["block2"]["number_of_columns"].item()) chunks = normalize_low_res_chunks( ("auto", "auto"), (nlines, ncols), From 3f552560d99d8209516cff4dfd033c15a11d9fab Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Tue, 12 Dec 2023 14:39:46 +0000 Subject: [PATCH 073/130] Repair AHI HSD tests. --- satpy/tests/reader_tests/test_ahi_hsd.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/satpy/tests/reader_tests/test_ahi_hsd.py b/satpy/tests/reader_tests/test_ahi_hsd.py index 2075b88947..7bf1562e1c 100644 --- a/satpy/tests/reader_tests/test_ahi_hsd.py +++ b/satpy/tests/reader_tests/test_ahi_hsd.py @@ -48,8 +48,8 @@ "compression_flag_for_data": 0, "hblock_number": 2, "number_of_bits_per_pixel": 16, - "number_of_columns": [11000], - "number_of_lines": [1100], + "number_of_columns": np.array([11000]), + "number_of_lines": np.array([1100]), "spare": "", } FAKE_PROJ_INFO: InfoDict = { @@ -135,8 +135,8 @@ def test_region(self, fromfile, np2str): "compression_flag_for_data": 0, "hblock_number": 2, "number_of_bits_per_pixel": 16, - "number_of_columns": [1000], - "number_of_lines": [1000], + "number_of_columns": np.array([1000]), + "number_of_lines": np.array([1000]), "spare": ""} area_def = fh.get_area_def(None) @@ -183,8 +183,8 @@ def test_segment(self, fromfile, np2str): "compression_flag_for_data": 0, "hblock_number": 2, "number_of_bits_per_pixel": 16, - "number_of_columns": [11000], - "number_of_lines": [1100], + "number_of_columns": np.array([11000]), + "number_of_lines": np.array([1100]), "spare": ""} area_def = fh.get_area_def(None) From 12192b1ac153a124892fa3a45dae112bbf027bbd Mon Sep 17 00:00:00 2001 From: Youva <120452807+YouvaEUMex@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:33:01 +0100 Subject: [PATCH 074/130] Update satpy/tests/reader_tests/test_fci_l2_nc.py Co-authored-by: Martin Raspaud --- satpy/tests/reader_tests/test_fci_l2_nc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index 7853cba900..e7a312d4b8 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -508,10 +508,10 @@ def test_byte_extraction(self): assert dataset.values == 0 -class TestFciL2NCAMVFileHandler(unittest.TestCase): +class TestFciL2NCAMVFileHandler: """Test the FciL2NCFileHandler reader.""" - def setUp(self): + def setup_method(self): """Set up the test by creating a test file and opening it with the reader.""" # Easiest way to test the reader is to create a test netCDF file on the fly # Create unique filenames to prevent race conditions when tests are run in parallel From 1b14e764212cf0af7076f3458593eeb9c7525aa1 Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Wed, 13 Dec 2023 13:46:08 +0000 Subject: [PATCH 075/130] Replace the reader for a lazy reader using cached_property decorator --- satpy/readers/fci_l2_nc.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/satpy/readers/fci_l2_nc.py b/satpy/readers/fci_l2_nc.py index 03f8e94f55..78020cdcf9 100644 --- a/satpy/readers/fci_l2_nc.py +++ b/satpy/readers/fci_l2_nc.py @@ -22,6 +22,7 @@ import xarray as xr from pyresample import geometry +from satpy._compat import cached_property from satpy.readers._geos_area import get_geos_area_naming, make_ext from satpy.readers.eum_base import get_service_mode from satpy.readers.file_handlers import BaseFileHandler @@ -153,6 +154,7 @@ def __init__(self, filename, filename_info, filetype_info, with_area_definition= self._projection = self.nc["mtg_geos_projection"] self.multi_dims = {"maximum_number_of_layers": "layer", "number_of_vis_channels": "vis_channel_id"} + def get_area_def(self, key): """Return the area definition.""" try: @@ -408,14 +410,15 @@ def __init__(self, filename, filename_info, filetype_info): """Open the NetCDF file with xarray and prepare for dataset reading.""" super().__init__(filename, filename_info, filetype_info) - # Use xarray's default netcdf4 engine to open the file - self.nc = xr.open_dataset( + @cached_property + def nc(self): + """Read the file.""" + return xr.open_dataset( self.filename, decode_cf=True, mask_and_scale=True, chunks={ "number_of_images": CHUNK_SIZE, - # 'number_of_height_estimates': CHUNK_SIZE, "number_of_winds": CHUNK_SIZE } ) From 6e3bc601515943e20d312fc5b01e0f5dc8d1bd80 Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Wed, 13 Dec 2023 14:29:06 +0000 Subject: [PATCH 076/130] Revert all attempt to introduce tmp_path pytest fixture --- satpy/tests/reader_tests/test_fci_l2_nc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index e7a312d4b8..44906c5040 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -509,7 +509,7 @@ def test_byte_extraction(self): assert dataset.values == 0 class TestFciL2NCAMVFileHandler: - """Test the FciL2NCFileHandler reader.""" + """Test the FciL2NCAMVFileHandler reader.""" def setup_method(self): """Set up the test by creating a test file and opening it with the reader.""" @@ -548,7 +548,8 @@ def setup_method(self): self.fh = FciL2NCAMVFileHandler(filename=self.test_file, filename_info={"channel":"test_channel"}, - filetype_info={}) + filetype_info={} + ) def tearDown(self): """Remove the previously created test file.""" From fe9c1db295d29585739fa1ba3f25a2b6933cb18e Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 07:34:16 +0000 Subject: [PATCH 077/130] Add tests for default functionality of HighCloudCompositor and LowCloudCompositor. From 30e89987c430d81a6236077e3a20acbbd9ad8704 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 07:44:35 +0000 Subject: [PATCH 078/130] Add tests for default functionality of HighCloudCompositor and LowCloudCompositor. --- satpy/tests/test_composites.py | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index 70bc2abf25..2fa87c1e1b 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -937,6 +937,57 @@ def test_call(self): np.testing.assert_allclose(res, exp) +class TestHighCloudCompositor: + """Test HighCloudCompositor.""" + + def setup_method(self): + """Create test data.""" + from pyresample.geometry import create_area_def + area = create_area_def(area_id="test", projection={"proj": "latlong"}, + center=(0, 45), width=3, height=3, resolution=35) + + self.data = xr.DataArray(da.from_array([[200, 250, 300], + [200, 250, 300], + [200, 250, 300]]), + dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}, + attrs={"area": area}) + + def test_default_behaviour(self): + """Test general default functionality of compositor.""" + from satpy.composites import HighCloudCompositor + with dask.config.set(scheduler=CustomScheduler(max_computes=1)): + comp = HighCloudCompositor(name="test") + res = comp([self.data]) + expexted_alpha = np.array([[1.0, 0.7142857, 0.0], + [1.0, 0.625, 0.0], + [1.0, 0.5555555, 0.0]]) + expected = np.stack([self.data, expexted_alpha]) + np.testing.assert_almost_equal(res.values, expected) + + +class TestLowCloudCompositor: + """Test LowCloudCompositor.""" + + def setup_method(self): + """Create test data.""" + self.btd = xr.DataArray(da.from_array([[0.0, 1.0, 10.0], [0.0, 1.0, 10.0], [0.0, 1.0, 10.0]]), + dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}) + self.bt_win = xr.DataArray(da.from_array([[250, 250, 250], [250, 250, 250], [150, 150, 150]]), + dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}) + self.lsm = xr.DataArray(da.from_array([[0., 0., 0.], [1., 1., 1.], [0., 1., 0.]]), + dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}) + + def test_default_behaviour(self): + """Test general default functionality of compositor.""" + from satpy.composites import LowCloudCompositor + with dask.config.set(scheduler=CustomScheduler(max_computes=1)): + comp = LowCloudCompositor(name="test") + res = comp([self.btd, self.bt_win, self.lsm]) + expexted_alpha = np.array([[0.0, 0.25, 1.0], [0.0, 0.25, 1.0], [0.0, 0.0, 0.0]]) + expected = np.stack([self.btd, expexted_alpha]) + np.testing.assert_equal(res.values, expected) + + class TestSingleBandCompositor(unittest.TestCase): """Test the single-band compositor.""" From 169bfbd7f8a19528d33172b2e05a034e88c1505e Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 07:50:44 +0000 Subject: [PATCH 079/130] Set alpha chanel to transparent instead of brightness temperature difference to zero when hiding potential IR3.8 channel noise for cold cloud tops. --- satpy/composites/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index bc5a199aa0..550040e5f5 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1202,11 +1202,6 @@ def __call__(self, projectables, **kwargs): lsm = lsm.squeeze(drop=True) lsm = lsm.round() # Make sure to have whole numbers in case of smearing from resampling - # Avoid spurious false alarms caused by noise in the 3.9um channel that can occur for very cold cloud tops - # TODO Consolidate this. Should it really be set to zero and thus within the threshold range? What if the - # lower threshold would be changed to -1 - btd = btd.where(bt_win >= 230, 0.0) - # Call CloudCompositor for land surface pixels self.transition_min, self.transition_max = self.range_land res = super().__call__([btd.where(lsm.isin(self.values_land))], **kwargs) @@ -1218,6 +1213,10 @@ def __call__(self, projectables, **kwargs): # Compine resutls for land and sea/water surface pixels res = res.where(lsm.isin(self.values_land), res_sea) + # Make pixels with cold window channel brightness temperatures transparent to avoid spurious false + # alarms caused by noise in the 3.9um channel that can occur for very cold cloud tops + res.loc["A"] = res.sel(bands="A").where(bt_win >= 230, 0.0) + return res From 130e768f6f5d5216aa2a8945d2bd8098bf61c39d Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 07:52:35 +0000 Subject: [PATCH 080/130] Implement keyword for alpha channel inversion in CloudCompositor and use for LowCloudCompositor. Remove corresponding alpha channel inversion in enhancement recipe. --- satpy/composites/__init__.py | 9 +++++++-- satpy/etc/enhancements/generic.yaml | 4 ---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 550040e5f5..9281dfae4f 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1015,7 +1015,7 @@ class CloudCompositor(GenericCompositor): """Detect clouds based on thresholding and use it as a mask for compositing.""" def __init__(self, name, transition_min=258.15, transition_max=298.15, - transition_gamma=3.0, **kwargs): + invert_alpha=False, transition_gamma=3.0, **kwargs): """Collect custom configuration values. Args: @@ -1028,6 +1028,7 @@ def __init__(self, name, transition_min=258.15, transition_max=298.15, """ self.transition_min = transition_min self.transition_max = transition_max + self.invert_alpha = invert_alpha self.transition_gamma = transition_gamma super(CloudCompositor, self).__init__(name, **kwargs) @@ -1050,6 +1051,9 @@ def __call__(self, projectables, **kwargs): alpha = alpha.where(data <= tr_max, 0.) alpha = alpha.where((data <= tr_min) | (data > tr_max), slope * data + offset) + if self.invert_alpha: + alpha = 1.0 - alpha + # gamma adjustment alpha **= gamma res = super(CloudCompositor, self).__call__((data, alpha), **kwargs) @@ -1155,6 +1159,7 @@ class LowCloudCompositor(CloudCompositor): def __init__(self, name, values_land=(1,), values_sea=(0,), range_land=(0.0, 4.0), range_sea=(0.0, 4.0), + invert_alpha=True, transition_gamma=1.0, **kwargs): """Init info. @@ -1182,7 +1187,7 @@ def __init__(self, name, values_land=(1,), values_sea=(0,), self.transition_gamma = transition_gamma self.transition_min = None # Placeholder for later use in CloudCompositor self.transition_max = None # Placeholder for later use in CloudCompositor - super().__init__(name, transition_gamma=transition_gamma, **kwargs) + super().__init__(name, invert_alpha=invert_alpha, transition_gamma=transition_gamma, **kwargs) def __call__(self, projectables, **kwargs): """Generate the composite. diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index efb9b2c6fa..dfd5b5f5c6 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -969,10 +969,6 @@ enhancements: geo_color_low_clouds: standard_name: geo_color_low_clouds operations: - - name: inverse - method: !!python/name:satpy.enhancements.invert - args: - - [False, True] - name: stretch method: !!python/name:satpy.enhancements.stretch kwargs: From 2da489b9c5d48ebf1e78b9faa0db3fa353848948 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Thu, 14 Dec 2023 08:55:56 +0100 Subject: [PATCH 081/130] hvplot tests Test for areadefinition data (single band and rgb). Test for swath data (only single band) --- satpy/tests/scene_tests/test_conversions.py | 47 +++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/satpy/tests/scene_tests/test_conversions.py b/satpy/tests/scene_tests/test_conversions.py index a886c3fa60..9b0dd9098e 100644 --- a/satpy/tests/scene_tests/test_conversions.py +++ b/satpy/tests/scene_tests/test_conversions.py @@ -81,6 +81,53 @@ def test_geoviews_basic_with_swath(self): # we assume that if we got something back, geoviews can use it assert gv_obj is not None + def test_hvplot_basic_with_area(self): + """Test converting a Scene to hvplot with a AreaDefinition.""" + from pyresample.geometry import AreaDefinition + scn = Scene() + area = AreaDefinition("test", "test", "test", + {"proj": "geos", "lon_0": -95.5, "h": 35786023.0}, + 2, 2, [-200, -200, 200, 200]) + scn["ds1"] = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=("y", "x"), + attrs={"start_time": datetime(2018, 1, 1), + "area": area, "units": "m"}) + hv_obj = scn.to_hvplot() + # we assume that if we got something back, hvplot can use it + assert hv_obj is not None + + def test_hvplot_rgb_with_area(self): + """Test converting a Scene to hvplot with a AreaDefinition.""" + from pyresample.geometry import AreaDefinition + scn = Scene() + area = AreaDefinition("test", "test", "test", + {"proj": "geos", "lon_0": -95.5, "h": 35786023.0}, + 2, 2, [-200, -200, 200, 200]) + scn["ds1"] = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=("y", "x"), + attrs={"start_time": datetime(2018, 1, 1), + "area": area, "units": "m"}) + scn["ds2"] = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=("y", "x"), + attrs={"start_time": datetime(2018, 1, 1), + "area": area, "units": "m"}) + scn["ds3"] = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=("y", "x"), + attrs={"start_time": datetime(2018, 1, 1), + "area": area, "units": "m"}) + hv_obj = scn.to_hvplot() + # we assume that if we got something back, hvplot can use it + assert hv_obj is not None + + def test_hvplot_basic_with_swath(self): + """Test converting a Scene to hvplot with a SwathDefinition.""" + from pyresample.geometry import SwathDefinition + scn = Scene() + longitude = xr.DataArray(da.zeros((2, 2))) + latitude = xr.DataArray(da.zeros((2, 2))) + area = SwathDefinition(longitude, latitude) + scn["ds1"] = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=("y", "x"), + attrs={"start_time": datetime(2018, 1, 1), + "area": area, "units": "m"}) + hv_obj = scn.to_hvplot() + # we assume that if we got something back, hvplot can use it + assert hv_obj is not None class TestToXarrayConversion: """Test Scene.to_xarray() conversion.""" From 56138aa7c9c5365b6ada8a367414d106d97f00ac Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 08:44:13 +0000 Subject: [PATCH 082/130] Fix alpha inversion to keep dataset attributes. --- satpy/composites/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 9281dfae4f..8089464be6 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1052,7 +1052,7 @@ def __call__(self, projectables, **kwargs): alpha = alpha.where((data <= tr_min) | (data > tr_max), slope * data + offset) if self.invert_alpha: - alpha = 1.0 - alpha + alpha.data = 1.0 - alpha.data # gamma adjustment alpha **= gamma From 43e6ce596e235dd5d2eea5e2e102e3c5c2813cf9 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 08:46:23 +0000 Subject: [PATCH 083/130] Move initialization of parent class variables to parent class init. --- satpy/composites/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 8089464be6..cb5a10663c 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1184,10 +1184,8 @@ def __init__(self, name, values_land=(1,), values_sea=(0,), self.values_sea = values_sea if type(values_sea) in [list, tuple] else [values_sea] self.range_land = range_land self.range_sea = range_sea - self.transition_gamma = transition_gamma - self.transition_min = None # Placeholder for later use in CloudCompositor - self.transition_max = None # Placeholder for later use in CloudCompositor - super().__init__(name, invert_alpha=invert_alpha, transition_gamma=transition_gamma, **kwargs) + super().__init__(name, transition_min=None, transition_max=None, + invert_alpha=invert_alpha, transition_gamma=transition_gamma, **kwargs) def __call__(self, projectables, **kwargs): """Generate the composite. From ffa15e1bb241ea7d2b4c599f22e82137ad0d7c41 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 08:48:22 +0000 Subject: [PATCH 084/130] Fix indentation. --- satpy/tests/test_composites.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index 2fa87c1e1b..6179fc8053 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -962,7 +962,7 @@ def test_default_behaviour(self): [1.0, 0.625, 0.0], [1.0, 0.5555555, 0.0]]) expected = np.stack([self.data, expexted_alpha]) - np.testing.assert_almost_equal(res.values, expected) + np.testing.assert_almost_equal(res.values, expected) class TestLowCloudCompositor: @@ -985,7 +985,7 @@ def test_default_behaviour(self): res = comp([self.btd, self.bt_win, self.lsm]) expexted_alpha = np.array([[0.0, 0.25, 1.0], [0.0, 0.25, 1.0], [0.0, 0.0, 0.0]]) expected = np.stack([self.btd, expexted_alpha]) - np.testing.assert_equal(res.values, expected) + np.testing.assert_equal(res.values, expected) class TestSingleBandCompositor(unittest.TestCase): From aae7c9ec28668074b89deab02e7b209a00964fcb Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 09:18:01 +0000 Subject: [PATCH 085/130] Move computation of expected data outside of the dask scheduler. --- satpy/tests/test_composites.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index 6179fc8053..bcda95aba8 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -946,9 +946,7 @@ def setup_method(self): area = create_area_def(area_id="test", projection={"proj": "latlong"}, center=(0, 45), width=3, height=3, resolution=35) - self.data = xr.DataArray(da.from_array([[200, 250, 300], - [200, 250, 300], - [200, 250, 300]]), + self.data = xr.DataArray(da.from_array([[200, 250, 300], [200, 250, 300], [200, 250, 300]]), dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}, attrs={"area": area}) @@ -958,11 +956,11 @@ def test_default_behaviour(self): with dask.config.set(scheduler=CustomScheduler(max_computes=1)): comp = HighCloudCompositor(name="test") res = comp([self.data]) - expexted_alpha = np.array([[1.0, 0.7142857, 0.0], - [1.0, 0.625, 0.0], - [1.0, 0.5555555, 0.0]]) - expected = np.stack([self.data, expexted_alpha]) - np.testing.assert_almost_equal(res.values, expected) + data = res.values + + expexted_alpha = np.array([[1.0, 0.7142857, 0.0], [1.0, 0.625, 0.0], [1.0, 0.5555555, 0.0]]) + expected = np.stack([self.data, expexted_alpha]) + np.testing.assert_almost_equal(data, expected) class TestLowCloudCompositor: @@ -983,9 +981,11 @@ def test_default_behaviour(self): with dask.config.set(scheduler=CustomScheduler(max_computes=1)): comp = LowCloudCompositor(name="test") res = comp([self.btd, self.bt_win, self.lsm]) - expexted_alpha = np.array([[0.0, 0.25, 1.0], [0.0, 0.25, 1.0], [0.0, 0.0, 0.0]]) - expected = np.stack([self.btd, expexted_alpha]) - np.testing.assert_equal(res.values, expected) + data = res.values + + expexted_alpha = np.array([[0.0, 0.25, 1.0], [0.0, 0.25, 1.0], [0.0, 0.0, 0.0]]) + expected = np.stack([self.btd, expexted_alpha]) + np.testing.assert_equal(data, expected) class TestSingleBandCompositor(unittest.TestCase): From 27358020edf6bd47579756b345c64676353e72c8 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 09:30:15 +0000 Subject: [PATCH 086/130] Add data type preservation tests for HighCloudCompositor and LowCloudCompositor. --- satpy/tests/test_composites.py | 48 ++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index bcda95aba8..db29a8572e 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -946,47 +946,67 @@ def setup_method(self): area = create_area_def(area_id="test", projection={"proj": "latlong"}, center=(0, 45), width=3, height=3, resolution=35) - self.data = xr.DataArray(da.from_array([[200, 250, 300], [200, 250, 300], [200, 250, 300]]), - dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}, - attrs={"area": area}) + self.data = xr.DataArray( + da.from_array(np.array([[200, 250, 300], [200, 250, 300], [200, 250, 300]], dtype=np.float32)), + dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}, + attrs={"area": area} + ) - def test_default_behaviour(self): + def test_high_cloud_compositor(self): """Test general default functionality of compositor.""" from satpy.composites import HighCloudCompositor with dask.config.set(scheduler=CustomScheduler(max_computes=1)): comp = HighCloudCompositor(name="test") res = comp([self.data]) data = res.values - expexted_alpha = np.array([[1.0, 0.7142857, 0.0], [1.0, 0.625, 0.0], [1.0, 0.5555555, 0.0]]) expected = np.stack([self.data, expexted_alpha]) np.testing.assert_almost_equal(data, expected) + def test_high_cloud_compositor_dtype(self): + """Test that the datatype is not altered by the compositor.""" + from satpy.composites import HighCloudCompositor + comp = HighCloudCompositor(name="test") + res = comp([self.data]) + assert res.data.dtype == np.float32 + class TestLowCloudCompositor: """Test LowCloudCompositor.""" def setup_method(self): """Create test data.""" - self.btd = xr.DataArray(da.from_array([[0.0, 1.0, 10.0], [0.0, 1.0, 10.0], [0.0, 1.0, 10.0]]), - dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}) - self.bt_win = xr.DataArray(da.from_array([[250, 250, 250], [250, 250, 250], [150, 150, 150]]), - dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}) - self.lsm = xr.DataArray(da.from_array([[0., 0., 0.], [1., 1., 1.], [0., 1., 0.]]), - dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}) - - def test_default_behaviour(self): + self.btd = xr.DataArray( + da.from_array(np.array([[0.0, 1.0, 10.0], [0.0, 1.0, 10.0], [0.0, 1.0, 10.0]], dtype=np.float32)), + dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]} + ) + self.bt_win = xr.DataArray( + da.from_array(np.array([[250, 250, 250], [250, 250, 250], [150, 150, 150]], dtype=np.float32)), + dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]} + ) + self.lsm = xr.DataArray( + da.from_array(np.array([[0., 0., 0.], [1., 1., 1.], [0., 1., 0.]], dtype=np.float32)), + dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]} + ) + + def test_low_cloud_compositor(self): """Test general default functionality of compositor.""" from satpy.composites import LowCloudCompositor with dask.config.set(scheduler=CustomScheduler(max_computes=1)): comp = LowCloudCompositor(name="test") res = comp([self.btd, self.bt_win, self.lsm]) data = res.values - expexted_alpha = np.array([[0.0, 0.25, 1.0], [0.0, 0.25, 1.0], [0.0, 0.0, 0.0]]) expected = np.stack([self.btd, expexted_alpha]) np.testing.assert_equal(data, expected) + def test_low_cloud_compositor_dtype(self): + """Test that the datatype is not altered by the compositor.""" + from satpy.composites import LowCloudCompositor + comp = LowCloudCompositor(name="test") + res = comp([self.btd, self.bt_win, self.lsm]) + assert res.data.dtype == np.float32 + class TestSingleBandCompositor(unittest.TestCase): """Test the single-band compositor.""" From 5e9763cabe573f968d5d6aafbe2c701b326be030 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 09:32:23 +0000 Subject: [PATCH 087/130] Move computation of data outside dask schedule and set max number of jobs to 0. --- satpy/tests/test_composites.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index db29a8572e..bf47c9ff9a 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -955,13 +955,12 @@ def setup_method(self): def test_high_cloud_compositor(self): """Test general default functionality of compositor.""" from satpy.composites import HighCloudCompositor - with dask.config.set(scheduler=CustomScheduler(max_computes=1)): + with dask.config.set(scheduler=CustomScheduler(max_computes=0)): comp = HighCloudCompositor(name="test") res = comp([self.data]) - data = res.values expexted_alpha = np.array([[1.0, 0.7142857, 0.0], [1.0, 0.625, 0.0], [1.0, 0.5555555, 0.0]]) expected = np.stack([self.data, expexted_alpha]) - np.testing.assert_almost_equal(data, expected) + np.testing.assert_almost_equal(res.values, expected) def test_high_cloud_compositor_dtype(self): """Test that the datatype is not altered by the compositor.""" @@ -992,13 +991,12 @@ def setup_method(self): def test_low_cloud_compositor(self): """Test general default functionality of compositor.""" from satpy.composites import LowCloudCompositor - with dask.config.set(scheduler=CustomScheduler(max_computes=1)): + with dask.config.set(scheduler=CustomScheduler(max_computes=0)): comp = LowCloudCompositor(name="test") res = comp([self.btd, self.bt_win, self.lsm]) - data = res.values expexted_alpha = np.array([[0.0, 0.25, 1.0], [0.0, 0.25, 1.0], [0.0, 0.0, 0.0]]) expected = np.stack([self.btd, expexted_alpha]) - np.testing.assert_equal(data, expected) + np.testing.assert_equal(res.values, expected) def test_low_cloud_compositor_dtype(self): """Test that the datatype is not altered by the compositor.""" From 6b3b40f0d11575266c1293aafcb8c1dde88c0e1e Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 09:53:07 +0000 Subject: [PATCH 088/130] Daskify computation of latitude array for HighCloudCompositor and preserve dtype. --- satpy/composites/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index cb5a10663c..9bb7043343 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1120,13 +1120,13 @@ def __call__(self, projectables, **kwargs): raise ValueError(f"Expected 1 dataset, got {len(projectables)}") data = projectables[0] - _, lats = data.attrs["area"].get_lonlats() + _, lats = data.attrs["area"].get_lonlats(chunks=data.chunks, dtype=data.dtype) lats = np.abs(lats) slope = (self.transition_min[1] - self.transition_min[0]) / (self.latitude_min[1] - self.latitude_min[0]) offset = self.transition_min[0] - slope * self.latitude_min[0] - tr_min_lat = xr.DataArray(name="tr_min_lat", coords=data.coords, dims=data.dims) + tr_min_lat = xr.DataArray(name="tr_min_lat", coords=data.coords, dims=data.dims).astype(data.dtype) tr_min_lat = tr_min_lat.where(lats >= self.latitude_min[0], self.transition_min[0]) tr_min_lat = tr_min_lat.where(lats <= self.latitude_min[1], self.transition_min[1]) tr_min_lat = tr_min_lat.where((lats < self.latitude_min[0]) | (lats > self.latitude_min[1]), From 48c1c63ef830421b631ed1a6e4aae5620ac0beba Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 09:56:53 +0000 Subject: [PATCH 089/130] use a common variable for testing dtype. --- satpy/tests/test_composites.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index bf47c9ff9a..f0515c0f93 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -945,9 +945,9 @@ def setup_method(self): from pyresample.geometry import create_area_def area = create_area_def(area_id="test", projection={"proj": "latlong"}, center=(0, 45), width=3, height=3, resolution=35) - + self.dtype = np.float32 self.data = xr.DataArray( - da.from_array(np.array([[200, 250, 300], [200, 250, 300], [200, 250, 300]], dtype=np.float32)), + da.from_array(np.array([[200, 250, 300], [200, 250, 300], [200, 250, 300]], dtype=self.dtype)), dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]}, attrs={"area": area} ) @@ -967,7 +967,7 @@ def test_high_cloud_compositor_dtype(self): from satpy.composites import HighCloudCompositor comp = HighCloudCompositor(name="test") res = comp([self.data]) - assert res.data.dtype == np.float32 + assert res.data.dtype == self.dtype class TestLowCloudCompositor: @@ -975,16 +975,17 @@ class TestLowCloudCompositor: def setup_method(self): """Create test data.""" + self.dtype = np.float32 self.btd = xr.DataArray( - da.from_array(np.array([[0.0, 1.0, 10.0], [0.0, 1.0, 10.0], [0.0, 1.0, 10.0]], dtype=np.float32)), + da.from_array(np.array([[0.0, 1.0, 10.0], [0.0, 1.0, 10.0], [0.0, 1.0, 10.0]], dtype=self.dtype)), dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]} ) self.bt_win = xr.DataArray( - da.from_array(np.array([[250, 250, 250], [250, 250, 250], [150, 150, 150]], dtype=np.float32)), + da.from_array(np.array([[250, 250, 250], [250, 250, 250], [150, 150, 150]], dtype=self.dtype)), dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]} ) self.lsm = xr.DataArray( - da.from_array(np.array([[0., 0., 0.], [1., 1., 1.], [0., 1., 0.]], dtype=np.float32)), + da.from_array(np.array([[0., 0., 0.], [1., 1., 1.], [0., 1., 0.]], dtype=self.dtype)), dims=("y", "x"), coords={"y": [0, 1, 2], "x": [0, 1, 2]} ) @@ -1003,7 +1004,7 @@ def test_low_cloud_compositor_dtype(self): from satpy.composites import LowCloudCompositor comp = LowCloudCompositor(name="test") res = comp([self.btd, self.bt_win, self.lsm]) - assert res.data.dtype == np.float32 + assert res.data.dtype == self.dtype class TestSingleBandCompositor(unittest.TestCase): From e9530eff917ac8d6e70c65c89f0810624d70cff4 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 09:57:45 +0000 Subject: [PATCH 090/130] Remove obsolete TODOs. --- satpy/composites/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 9bb7043343..034e8a7821 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1114,8 +1114,6 @@ def __call__(self, projectables, **kwargs): `projectables` is expected to be a list or tuple with a single element: - index 0: Brightness temperature of a thermal infrared window channel (e.g. 10.5 microns). """ - # TODO Optimize and make sure that there are no early unnecessary dask computations. Is there a way to avoid - # computation of the latitude array? if len(projectables) != 1: raise ValueError(f"Expected 1 dataset, got {len(projectables)}") @@ -1196,7 +1194,6 @@ def __call__(self, projectables, **kwargs): - index 1. Brightness temperature of the window channel (used to filter out noise-induced false alarms). - index 2: Land-Sea-Mask. """ - # TODO Optimize and make sure that there are no early unnecessary dask computations if len(projectables) != 3: raise ValueError(f"Expected 3 datasets, got {len(projectables)}") From e6dc6165a4115191d8637b21b09467ab5db04681 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 12:16:19 +0000 Subject: [PATCH 091/130] Modify tests for NDVIHybridGreen compositor to assert 0 dask computations in compositor code. --- satpy/tests/compositor_tests/test_spectral.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/satpy/tests/compositor_tests/test_spectral.py b/satpy/tests/compositor_tests/test_spectral.py index e46cff4d0c..7472016c00 100644 --- a/satpy/tests/compositor_tests/test_spectral.py +++ b/satpy/tests/compositor_tests/test_spectral.py @@ -97,7 +97,7 @@ def setup_method(self): def test_ndvi_hybrid_green(self): """Test General functionality with linear scaling from ndvi to blend fraction.""" - with dask.config.set(scheduler=CustomScheduler(max_computes=1)): + with dask.config.set(scheduler=CustomScheduler(max_computes=0)): comp = NDVIHybridGreen("ndvi_hybrid_green", limits=(0.15, 0.05), prerequisites=(0.51, 0.65, 0.85), standard_name="toa_bidirectional_reflectance") @@ -107,12 +107,12 @@ def test_ndvi_hybrid_green(self): assert isinstance(res.data, da.Array) assert res.attrs["name"] == "ndvi_hybrid_green" assert res.attrs["standard_name"] == "toa_bidirectional_reflectance" - data = res.values + data = res.values np.testing.assert_array_almost_equal(data, np.array([[0.2633, 0.3071], [0.2115, 0.3420]]), decimal=4) def test_ndvi_hybrid_green_dtype(self): """Test that the datatype is not altered by the compositor.""" - with dask.config.set(scheduler=CustomScheduler(max_computes=1)): + with dask.config.set(scheduler=CustomScheduler(max_computes=0)): comp = NDVIHybridGreen("ndvi_hybrid_green", limits=(0.15, 0.05), prerequisites=(0.51, 0.65, 0.85), standard_name="toa_bidirectional_reflectance") res = comp((self.c01, self.c02, self.c03)).compute() @@ -120,15 +120,15 @@ def test_ndvi_hybrid_green_dtype(self): def test_nonlinear_scaling(self): """Test non-linear scaling using `strength` term.""" - with dask.config.set(scheduler=CustomScheduler(max_computes=1)): + with dask.config.set(scheduler=CustomScheduler(max_computes=0)): comp = NDVIHybridGreen("ndvi_hybrid_green", limits=(0.15, 0.05), strength=2.0, prerequisites=(0.51, 0.65, 0.85), standard_name="toa_bidirectional_reflectance") res = comp((self.c01, self.c02, self.c03)) - res_np = res.data.compute() - assert res.dtype == res_np.dtype - assert res.dtype == np.float32 + res_np = res.data.compute() + assert res.dtype == res_np.dtype + assert res.dtype == np.float32 np.testing.assert_array_almost_equal(res.data, np.array([[0.2646, 0.3075], [0.2120, 0.3471]]), decimal=4) def test_invalid_strength(self): From 3259552f53e90b19e132acc1e8ce8a5e1144ef99 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 12:24:16 +0000 Subject: [PATCH 092/130] Fix typos. --- satpy/composites/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index a7cc3d6e1c..09985d6ba1 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1079,7 +1079,7 @@ class HighCloudCompositor(CloudCompositor): - transition_min = transition_min[0] where abs(latitude) < latitude_min(0) - transition_min = transition_min[1] where abs(latitude) > latitude_min(0) - - transition_min = linear interpolation between transition_min[0] and transition_min[1] as a funtion + - transition_min = linear interpolation between transition_min[0] and transition_min[1] as a function of where abs(latitude). """ @@ -1139,7 +1139,7 @@ def __call__(self, projectables, **kwargs): class LowCloudCompositor(CloudCompositor): """Detect low-level clouds based on thresholding and use it as a mask for compositing during night-time. - This compsitor computes the brightness temperature difference between a window channel (e.g. 10.5 micron) + This compositor computes the brightness temperature difference between a window channel (e.g. 10.5 micron) and the near-infrared channel e.g. (3.8 micron) and uses this brightness temperature difference, `BTD`, to create a partially transparent mask for compositing. From 1bb565587ca5cd8ff845ff4d4de5045bb052ae8d Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 14 Dec 2023 15:03:55 +0200 Subject: [PATCH 093/130] Replace GradientSearchResampler with the helper method in dosctring --- satpy/resample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/resample.py b/satpy/resample.py index 336e3fec11..8b8f67dabf 100644 --- a/satpy/resample.py +++ b/satpy/resample.py @@ -42,7 +42,7 @@ "bucket_sum", "Sum Bucket Resampling", :class:`~satpy.resample.BucketSum` "bucket_count", "Count Bucket Resampling", :class:`~satpy.resample.BucketCount` "bucket_fraction", "Fraction Bucket Resampling", :class:`~satpy.resample.BucketFraction` - "gradient_search", "Gradient Search Resampling", :class:`~pyresample.gradient.GradientSearchResampler` + "gradient_search", "Gradient Search Resampling", :meth:`~pyresample.gradient.create_gradient_search_resampler` The resampling algorithm used can be specified with the ``resampler`` keyword argument and defaults to ``nearest``: From 5c49858463b1a4d9fa4dd93312923278e3ba7aa9 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 14 Dec 2023 14:46:08 +0100 Subject: [PATCH 094/130] Use fixtuers for test AMV file --- satpy/tests/reader_tests/test_fci_l2_nc.py | 113 ++++++++++----------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/satpy/tests/reader_tests/test_fci_l2_nc.py b/satpy/tests/reader_tests/test_fci_l2_nc.py index 44906c5040..84681b0f02 100644 --- a/satpy/tests/reader_tests/test_fci_l2_nc.py +++ b/satpy/tests/reader_tests/test_fci_l2_nc.py @@ -508,66 +508,65 @@ def test_byte_extraction(self): assert dataset.values == 0 -class TestFciL2NCAMVFileHandler: - """Test the FciL2NCAMVFileHandler reader.""" - - def setup_method(self): - """Set up the test by creating a test file and opening it with the reader.""" - # Easiest way to test the reader is to create a test netCDF file on the fly - # Create unique filenames to prevent race conditions when tests are run in parallel - self.test_file = str(uuid.uuid4()) + ".nc" - with Dataset(self.test_file, "w") as nc: - # Create dimensions - nc.createDimension("number_of_winds", 50000) - - # add global attributes - nc.data_source = "test_data_source" - nc.platform = "test_platform" - - # Add datasets - latitude = nc.createVariable("latitude", np.float32, dimensions=("number_of_winds",)) - latitude[:] = np.arange(50000) - - longitude = nc.createVariable("y", np.float32, dimensions=("number_of_winds",)) - longitude[:] = np.arange(50000) - - qi = nc.createVariable("product_quality", np.int8) - qi[:] = 99. - test_dataset = nc.createVariable("test_dataset", np.float32, - dimensions="number_of_winds") - test_dataset[:] = np.ones(50000) - test_dataset.test_attr = "attr" - test_dataset.units = "test_units" - - mtg_geos_projection = nc.createVariable("mtg_geos_projection", int, dimensions=()) - mtg_geos_projection.longitude_of_projection_origin = 0.0 - mtg_geos_projection.semi_major_axis = 6378137. - mtg_geos_projection.inverse_flattening = 298.257223563 - mtg_geos_projection.perspective_point_height = 35786400. +@pytest.fixture(scope="module") +def amv_file(tmp_path_factory): + """Create an AMV file.""" + test_file = tmp_path_factory.mktemp("data") / "fci_l2_amv.nc" + + with Dataset(test_file, "w") as nc: + # Create dimensions + nc.createDimension("number_of_winds", 50000) + + # add global attributes + nc.data_source = "test_data_source" + nc.platform = "test_platform" + + # Add datasets + latitude = nc.createVariable("latitude", np.float32, dimensions=("number_of_winds",)) + latitude[:] = np.arange(50000) + + longitude = nc.createVariable("y", np.float32, dimensions=("number_of_winds",)) + longitude[:] = np.arange(50000) + + qi = nc.createVariable("product_quality", np.int8) + qi[:] = 99. + + test_dataset = nc.createVariable("test_dataset", np.float32, + dimensions="number_of_winds") + test_dataset[:] = np.ones(50000) + test_dataset.test_attr = "attr" + test_dataset.units = "test_units" + + mtg_geos_projection = nc.createVariable("mtg_geos_projection", int, dimensions=()) + mtg_geos_projection.longitude_of_projection_origin = 0.0 + mtg_geos_projection.semi_major_axis = 6378137. + mtg_geos_projection.inverse_flattening = 298.257223563 + mtg_geos_projection.perspective_point_height = 35786400. + return test_file + + +@pytest.fixture(scope="module") +def amv_filehandler(amv_file): + """Create an AMV filehandler.""" + return FciL2NCAMVFileHandler(filename=amv_file, + filename_info={"channel":"test_channel"}, + filetype_info={} + ) - self.fh = FciL2NCAMVFileHandler(filename=self.test_file, - filename_info={"channel":"test_channel"}, - filetype_info={} - ) - def tearDown(self): - """Remove the previously created test file.""" - # First delete the file handler, forcing the file to be closed if still open - del self.fh - # Then we can safely remove the file from the system - with suppress(OSError): - os.remove(self.test_file) +class TestFciL2NCAMVFileHandler: + """Test the FciL2NCAMVFileHandler reader.""" - def test_all_basic(self): + def test_all_basic(self, amv_filehandler, amv_file): """Test all basic functionalities.""" - assert self.fh.spacecraft_name == "test_platform" - assert self.fh.sensor_name == "test_data_source" - assert self.fh.ssp_lon == 0.0 + assert amv_filehandler.spacecraft_name == "test_platform" + assert amv_filehandler.sensor_name == "test_data_source" + assert amv_filehandler.ssp_lon == 0.0 - global_attributes = self.fh._get_global_attributes() + global_attributes = amv_filehandler._get_global_attributes() expected_global_attributes = { - "filename": self.test_file, + "filename": amv_file, "spacecraft_name": "test_platform", "sensor": "test_data_source", "platform_name": "test_platform", @@ -575,9 +574,9 @@ def test_all_basic(self): } assert global_attributes == expected_global_attributes - def test_dataset(self): + def test_dataset(self, amv_filehandler): """Test the correct execution of the get_dataset function with a valid file_key.""" - dataset = self.fh.get_dataset(make_dataid(name="test_dataset", resolution=2000), + dataset = amv_filehandler.get_dataset(make_dataid(name="test_dataset", resolution=2000), {"name": "test_dataset", "file_key": "test_dataset", "fill_value": -999, @@ -587,9 +586,9 @@ def test_dataset(self): assert dataset.attrs["units"] == "test_units" assert dataset.attrs["fill_value"] == -999 - def test_dataset_with_invalid_filekey(self): + def test_dataset_with_invalid_filekey(self, amv_filehandler): """Test the correct execution of the get_dataset function with an invalid file_key.""" - invalid_dataset = self.fh.get_dataset(make_dataid(name="test_invalid", resolution=2000), + invalid_dataset = amv_filehandler.get_dataset(make_dataid(name="test_invalid", resolution=2000), {"name": "test_invalid", "file_key": "test_invalid", "fill_value": -999, From 55d3622c343a3a5923b622318c71a1073aa796b5 Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 14 Dec 2023 16:07:56 +0100 Subject: [PATCH 095/130] add failing test for unsorted segments --- satpy/tests/test_yaml_reader.py | 35 +++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/satpy/tests/test_yaml_reader.py b/satpy/tests/test_yaml_reader.py index 41439a1ac6..3f1db6a977 100644 --- a/satpy/tests/test_yaml_reader.py +++ b/satpy/tests/test_yaml_reader.py @@ -971,10 +971,11 @@ def _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_p get_segment_position_info = MagicMock() get_segment_position_info.return_value = chk_pos_info - fh = MagicMock() filetype_info = {"expected_segments": expected_segments, "file_type": "filetype1"} filename_info = {"segment": segment} + + fh = _create_mocked_basic_fh() fh.filetype_info = filetype_info fh.filename_info = filename_info fh.get_area_def = get_area_def @@ -983,6 +984,12 @@ def _create_mocked_fh_and_areadef(aex, ashape, expected_segments, segment, chk_p return fh, seg_area +def _create_mocked_basic_fh(): + fake_fh = MagicMock() + fake_fh.filename_info = {} + fake_fh.filetype_info = {} + return fake_fh + class TestGEOSegmentYAMLReader(unittest.TestCase): """Test GEOSegmentYAMLReader.""" @@ -993,9 +1000,7 @@ def test_get_expected_segments(self, cfh): from satpy.readers.yaml_reader import GEOSegmentYAMLReader reader = GEOSegmentYAMLReader() - fake_fh = MagicMock() - fake_fh.filename_info = {} - fake_fh.filetype_info = {} + fake_fh = _create_mocked_basic_fh() cfh.return_value = {"ft1": [fake_fh]} # default (1) @@ -1030,6 +1035,28 @@ def test_get_expected_segments(self, cfh): es = created_fhs["ft1"][0].filename_info["segment"] assert es == 5 + @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) + @patch.object(yr.FileYAMLReader, "create_filehandlers") + def test_segments_sorting(self, cfh): + """Test that segment filehandlers are sorted by segment number.""" + from satpy.readers.yaml_reader import GEOSegmentYAMLReader + reader = GEOSegmentYAMLReader() + + # create filehandlers with different segment numbers + fake_fh_1 = _create_mocked_basic_fh() + fake_fh_1.filename_info["segment"] = 1 + fake_fh_2 = _create_mocked_basic_fh() + fake_fh_2.filename_info["segment"] = 2 + fake_fh_3 = _create_mocked_basic_fh() + fake_fh_3.filename_info["segment"] = 3 + + # put the filehandlers in an unsorted order + cfh.return_value = {"ft1": [fake_fh_1, fake_fh_3, fake_fh_2]} + + # check that the created filehandlers are sorted by segment number + created_fhs = reader.create_filehandlers(["fake.nc"]) + assert [fh.filename_info["segment"] for fh in created_fhs["ft1"]] == [1, 2, 3] + @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) @patch("satpy.readers.yaml_reader.FileYAMLReader._load_dataset") @patch("satpy.readers.yaml_reader.xr") From be666eb1aa6c0f5dd763e42787e4b264bd7ac2e3 Mon Sep 17 00:00:00 2001 From: youva Aoun Date: Thu, 14 Dec 2023 15:12:24 +0000 Subject: [PATCH 096/130] Fix name for amv lat/lon to avoid dupliacte in the yaml --- satpy/etc/readers/fci_l2_nc.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/etc/readers/fci_l2_nc.yaml b/satpy/etc/readers/fci_l2_nc.yaml index f9c12849eb..1ad5d576a0 100644 --- a/satpy/etc/readers/fci_l2_nc.yaml +++ b/satpy/etc/readers/fci_l2_nc.yaml @@ -2837,13 +2837,13 @@ datasets: file_key: channel_id standard_name: channel_id - latitude: + amv_latitude: name: latitude file_type: nc_fci_amv file_key: latitude standard_name: latitude - longitude: + amv_longitude: name: longitude file_type: nc_fci_amv file_key: longitude From 27041a451f3e6ca4933f75a7167ab272ae23dc07 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Thu, 14 Dec 2023 16:32:53 +0100 Subject: [PATCH 097/130] add control for swath data --- satpy/scene.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index e2d292a992..aea5b44cfe 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -39,10 +39,10 @@ from satpy.utils import convert_remote_files_to_fsspec, get_storage_options_from_reader_kwargs from satpy.writers import load_writer -try: - import hvplot.xarray as hvplot_xarray # noqa -except ImportError: - hvplot_xarray = None +#try: +# import hvplot.xarray as hvplot_xarray # noqa +#except ImportError: +# hvplot_xarray = None LOG = logging.getLogger(__name__) @@ -1092,6 +1092,7 @@ def to_hvplot(self, datasets=None, *args, **kwargs): plot.ash+plot.IR_108 """ + def _get_crs(xarray_ds): return xarray_ds.area.to_cartopy_crs() @@ -1112,17 +1113,27 @@ def _plot_quadmesh(xarray_ds, variable, **defaults): clabel=f"[{_get_units(xarray_ds,variable)}]", title=title, **defaults) - if hvplot_xarray is None: - raise ImportError("'hvplot' must be installed to use this feature") + #def _check_hvplot_library(): + # if hvplot_xarray is None: + # raise ImportError("'hvplot' must be installed to use this feature") +# +# _check_hvplot_library() plot = Overlay() xarray_ds = self.to_xarray_dataset(datasets) - ccrs = _get_crs(xarray_ds) + + if hasattr(xarray_ds, "area") and hasattr(xarray_ds.area, "to_cartopy_crs"): + ccrs = _get_crs(xarray_ds) + defaults={"x":"x","y":"y"} + else: + ccrs = None + defaults={"x":"longitude","y":"latitude"} + if datasets is None: datasets = list(xarray_ds.keys()) - defaults = dict(x="x", y="y", data_aspect=1, project=True, geo=True, + defaults.update(data_aspect=1, project=True, geo=True, crs=ccrs, projection=ccrs, rasterize=True, coastline="110m", cmap="Plasma", responsive=True, dynamic=False, framewise=True, colorbar=False, global_extent=False, xlabel="Longitude", @@ -1130,6 +1141,9 @@ def _plot_quadmesh(xarray_ds, variable, **defaults): defaults.update(kwargs) + #if "latitude" in xarray_ds.coords: + # defaults.update({"x":"longitude","y":"latitude"}) + for element in datasets: title = f"{element} @ {_get_timestamp(xarray_ds)}" if xarray_ds[element].shape[0] == 3: From 9dd1b28e7f9b9086a9c9872d75c17e03abee544e Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 14 Dec 2023 16:43:58 +0100 Subject: [PATCH 098/130] move sorting from _load_dataset to create_filehandlers so that it acts also when pad_data is False --- satpy/readers/yaml_reader.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index ff3599052a..73b5f7b6ee 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -1157,7 +1157,13 @@ class GEOSegmentYAMLReader(GEOFlippableFileYAMLReader): """ def create_filehandlers(self, filenames, fh_kwargs=None): - """Create file handler objects and determine expected segments for each.""" + """Create file handler objects and determine expected segments for each. + + Additionally, sort the filehandlers by segment number to avoid + issues with filenames where start_time or alphabetic sorting does not + produce the correct order. + + """ created_fhs = super(GEOSegmentYAMLReader, self).create_filehandlers( filenames, fh_kwargs=fh_kwargs) @@ -1171,6 +1177,11 @@ def create_filehandlers(self, filenames, fh_kwargs=None): # add segment key-values for FCI filehandlers if "segment" not in fh.filename_info: fh.filename_info["segment"] = fh.filename_info.get("count_in_repeat_cycle", 1) + + # sort by segment number + for file_type in created_fhs.keys(): + created_fhs[file_type] = sorted(created_fhs[file_type], key=lambda x: x.filename_info.get("segment", 1)) + return created_fhs def _load_dataset(self, dsid, ds_info, file_handlers, dim="y", pad_data=True): @@ -1322,11 +1333,9 @@ def _find_missing_segments(file_handlers, ds_info, dsid): failure = True counter = 1 expected_segments = 1 - # get list of file handlers in segment order - # (ex. first segment, second segment, etc) - handlers = sorted(file_handlers, key=lambda x: x.filename_info.get("segment", 1)) + projectable = None - for fh in handlers: + for fh in file_handlers: if fh.filetype_info["file_type"] in ds_info["file_type"]: expected_segments = fh.filetype_info["expected_segments"] From 8b9c46e6e895cd98c0347b5ef9ae45334853bb65 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 16:08:22 +0000 Subject: [PATCH 099/130] Fix bug related to initial variables being overwritten and later used again with wrong dimensions. Add unit test to catch this. --- satpy/composites/__init__.py | 44 ++++++++++++++++++---------------- satpy/tests/test_composites.py | 8 +++++++ 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 09985d6ba1..4153a85963 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1083,30 +1083,31 @@ class HighCloudCompositor(CloudCompositor): of where abs(latitude). """ - def __init__(self, name, transition_min=(210., 230.), transition_max=300, latitude_min=(30., 60.), # noqa: D417 - transition_gamma=1.0, **kwargs): + def __init__(self, name, transition_min_limits=(210., 230.), latitude_min_limits=(30., 60.), # noqa: D417 + transition_max=300, transition_gamma=1.0, **kwargs): """Collect custom configuration values. Args: - transition_min (tuple): Brightness temperature values used to identify opaque white - clouds at different latitudes + transition_min_limits (tuple): Brightness temperature values used to identify opaque white + clouds at different latitudes transition_max (float): Brightness temperatures above this value are not considered to be high clouds -> transparent - latitude_min (tuple): Latitude values defining the intervals for computing latitude-dependent - transition_min values. + latitude_min_limits (tuple): Latitude values defining the intervals for computing latitude-dependent + `transition_min` values from `transition_min_limits`. transition_gamma (float): Gamma correction to apply to the alpha channel within the brightness temperature range (`transition_min` to `transition_max`). """ - if len(transition_min) != 2: - raise ValueError(f"Expected 2 `transition_min` values, got {len(transition_min)}") - if len(latitude_min) != 2: - raise ValueError(f"Expected 2 `latitude_min` values, got {len(latitude_min)}") + if len(transition_min_limits) != 2: + raise ValueError(f"Expected 2 `transition_min_limits` values, got {len(transition_min_limits)}") + if len(transition_min_limits) != 2: + raise ValueError(f"Expected 2 `latitude_min_limits` values, got {len(transition_min_limits)}") if type(transition_max) in [list, tuple]: raise ValueError(f"Expected `transition_max` to be of type float, is of type {type(transition_max)}") - self.latitude_min = latitude_min - super().__init__(name, transition_min=transition_min, transition_max=transition_max, + self.transition_min_limits = transition_min_limits + self.latitude_min_limits = latitude_min_limits + super().__init__(name, transition_min=None, transition_max=transition_max, transition_gamma=transition_gamma, **kwargs) def __call__(self, projectables, **kwargs): @@ -1122,16 +1123,17 @@ def __call__(self, projectables, **kwargs): _, lats = data.attrs["area"].get_lonlats(chunks=data.chunks, dtype=data.dtype) lats = np.abs(lats) - slope = (self.transition_min[1] - self.transition_min[0]) / (self.latitude_min[1] - self.latitude_min[0]) - offset = self.transition_min[0] - slope * self.latitude_min[0] - - tr_min_lat = xr.DataArray(name="tr_min_lat", coords=data.coords, dims=data.dims).astype(data.dtype) - tr_min_lat = tr_min_lat.where(lats >= self.latitude_min[0], self.transition_min[0]) - tr_min_lat = tr_min_lat.where(lats <= self.latitude_min[1], self.transition_min[1]) - tr_min_lat = tr_min_lat.where((lats < self.latitude_min[0]) | (lats > self.latitude_min[1]), - slope * lats + offset) + slope = (self.transition_min_limits[1] - self.transition_min_limits[0]) / \ + (self.latitude_min_limits[1] - self.latitude_min_limits[0]) + offset = self.transition_min_limits[0] - slope * self.latitude_min_limits[0] - self.transition_min = tr_min_lat + # Compute pixel-level latitude dependent transition_min values and pass to parent CloudCompositor class + transition_min = xr.DataArray(name="transition_min", coords=data.coords, dims=data.dims).astype(data.dtype) + transition_min = transition_min.where(lats >= self.latitude_min_limits[0], self.transition_min_limits[0]) + transition_min = transition_min.where(lats <= self.latitude_min_limits[1], self.transition_min_limits[1]) + transition_min = transition_min.where((lats < self.latitude_min_limits[0]) | + (lats > self.latitude_min_limits[1]), slope * lats + offset) + self.transition_min = transition_min return super().__call__(projectables, **kwargs) diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index 6b79a06a99..aa4f56f9de 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -973,6 +973,14 @@ def test_high_cloud_compositor(self): expected = np.stack([self.data, expexted_alpha]) np.testing.assert_almost_equal(res.values, expected) + def test_high_cloud_compositor_multiple_calls(self): + """Test that the modified init variables are reset properly when calling the compositor multiple times.""" + from satpy.composites import HighCloudCompositor + comp = HighCloudCompositor(name="test") + res = comp([self.data]) + res2 = comp([self.data]) + np.testing.assert_equal(res.values, res2.values) + def test_high_cloud_compositor_dtype(self): """Test that the datatype is not altered by the compositor.""" from satpy.composites import HighCloudCompositor From ef3ebfc7fc5a65e39a60a8d5b8e3242188941335 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 16:56:57 +0000 Subject: [PATCH 100/130] Use crude stretch instead in order to increase image contrast of especially cold cloud tops. --- satpy/etc/enhancements/generic.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index dfd5b5f5c6..25680d6db9 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -964,7 +964,7 @@ enhancements: - name: stretch method: !!python/name:satpy.enhancements.stretch kwargs: - stretch: linear + stretch: crude geo_color_low_clouds: standard_name: geo_color_low_clouds @@ -972,7 +972,7 @@ enhancements: - name: stretch method: !!python/name:satpy.enhancements.stretch kwargs: - stretch: linear + stretch: crude - name: colorize method: !!python/name:satpy.enhancements.colorize kwargs: From eaa24be4804d292482f855ae64f0070b6ec138ad Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Thu, 14 Dec 2023 16:58:05 +0000 Subject: [PATCH 101/130] Refine blending range for DayNightCompositor. --- satpy/etc/composites/abi.yaml | 4 ++-- satpy/etc/composites/ahi.yaml | 4 ++-- satpy/etc/composites/fci.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/satpy/etc/composites/abi.yaml b/satpy/etc/composites/abi.yaml index e950ba027f..b40f353e6a 100644 --- a/satpy/etc/composites/abi.yaml +++ b/satpy/etc/composites/abi.yaml @@ -756,8 +756,8 @@ composites: # GeoColor geo_color: compositor: !!python/name:satpy.composites.DayNightCompositor - lim_low: 73 - lim_high: 82 + lim_low: 78 + lim_high: 88 standard_name: geo_color_day_night_blend prerequisites: - true_color diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index e088bcf1a6..066b9cf0f2 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -506,8 +506,8 @@ composites: geo_color: compositor: !!python/name:satpy.composites.DayNightCompositor - lim_low: 73 - lim_high: 82 + lim_low: 78 + lim_high: 88 standard_name: geo_color_day_night_blend prerequisites: - true_color_ndvi_green diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index d564cc1f36..f27011d301 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -128,8 +128,8 @@ composites: # GeoColor geo_color: compositor: !!python/name:satpy.composites.DayNightCompositor - lim_low: 73 - lim_high: 82 + lim_low: 78 + lim_high: 88 standard_name: geo_color_day_night_blend prerequisites: - true_color From 739ce460f16550be0b2428109eb8e49d049707f0 Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 14 Dec 2023 18:01:14 +0100 Subject: [PATCH 102/130] sort the reader attribute file_handlers instead of the returned created filehandlers and change test accordingly --- satpy/readers/yaml_reader.py | 10 ++++++---- satpy/tests/test_yaml_reader.py | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index 73b5f7b6ee..3171449b03 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -1178,12 +1178,14 @@ def create_filehandlers(self, filenames, fh_kwargs=None): if "segment" not in fh.filename_info: fh.filename_info["segment"] = fh.filename_info.get("count_in_repeat_cycle", 1) - # sort by segment number - for file_type in created_fhs.keys(): - created_fhs[file_type] = sorted(created_fhs[file_type], key=lambda x: x.filename_info.get("segment", 1)) - + self._sort_segment_filehandler_by_segment_number() return created_fhs + def _sort_segment_filehandler_by_segment_number(self): + for file_type in self.file_handlers.keys(): + self.file_handlers[file_type] = sorted(self.file_handlers[file_type], + key=lambda x: x.filename_info.get("segment", 0)) + def _load_dataset(self, dsid, ds_info, file_handlers, dim="y", pad_data=True): """Load only a piece of the dataset.""" if not pad_data: diff --git a/satpy/tests/test_yaml_reader.py b/satpy/tests/test_yaml_reader.py index 3f1db6a977..0b0293e453 100644 --- a/satpy/tests/test_yaml_reader.py +++ b/satpy/tests/test_yaml_reader.py @@ -1051,11 +1051,11 @@ def test_segments_sorting(self, cfh): fake_fh_3.filename_info["segment"] = 3 # put the filehandlers in an unsorted order - cfh.return_value = {"ft1": [fake_fh_1, fake_fh_3, fake_fh_2]} + reader.file_handlers = {"ft1": [fake_fh_1, fake_fh_3, fake_fh_2]} # check that the created filehandlers are sorted by segment number - created_fhs = reader.create_filehandlers(["fake.nc"]) - assert [fh.filename_info["segment"] for fh in created_fhs["ft1"]] == [1, 2, 3] + reader.create_filehandlers(["fake.nc"]) + assert [fh.filename_info["segment"] for fh in reader.file_handlers["ft1"]] == [1, 2, 3] @patch.object(yr.FileYAMLReader, "__init__", lambda x: None) @patch("satpy.readers.yaml_reader.FileYAMLReader._load_dataset") From 203acca9e8261028af826a1050a89241f9113956 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 14 Dec 2023 14:07:46 -0600 Subject: [PATCH 103/130] Fix composites failing on non-aligned geolocation coordinates --- satpy/composites/__init__.py | 18 ++++++++++++++++++ satpy/tests/test_composites.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 5fa9ca575b..a0ce73caea 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -185,6 +185,7 @@ def match_data_arrays(self, data_arrays): """ self.check_geolocation(data_arrays) new_arrays = self.drop_coordinates(data_arrays) + new_arrays = self.align_geo_coordinates(new_arrays) new_arrays = list(unify_chunks(*new_arrays)) return new_arrays @@ -210,6 +211,23 @@ def drop_coordinates(self, data_arrays): return new_arrays + def align_geo_coordinates(self, data_arrays: Sequence[xr.DataArray]) -> list[xr.DataArray]: + """Align DataArrays along geolocation coordinates. + + See :func:`~xarray.align` for more information. This function uses + the "override" join method to essentially ignore differences between + coordinates. The :meth:`check_geolocation` should be called before + this to ensure that geolocation coordinates and "area" are compatible. + The :meth:`drop_coordinates` method should be called before this to + ensure that coordinates that are considered "negligible" when computing + composites do not affect alignment. + + """ + non_geo_coords = tuple( + coord_name for data_arr in data_arrays + for coord_name in data_arr.coords if coord_name not in ("x", "y")) + return xr.align(*data_arrays, join="override", exclude=non_geo_coords) + def check_geolocation(self, data_arrays): """Check that the geolocations of the *data_arrays* are compatible. diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index 4a7b2a2ce9..830a427c4a 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -37,7 +37,7 @@ # - tmp_path -class TestMatchDataArrays(unittest.TestCase): +class TestMatchDataArrays: """Test the utility method 'match_data_arrays'.""" def _get_test_ds(self, shape=(50, 100), dims=("y", "x")): @@ -132,6 +132,38 @@ def test_nondimensional_coords(self): ret_datasets = comp.match_data_arrays([ds, ds]) assert "acq_time" not in ret_datasets[0].coords + def test_almost_equal_geo_coordinates(self): + """Test that coordinates that are almost-equal still match. + + See https://github.com/pytroll/satpy/issues/2668 for discussion. + + Various operations like cropping and resampling can cause + geo-coordinates (y, x) to be very slightly unequal due to floating + point precision. This test makes sure that even in those cases we + can still generate composites from DataArrays with these coordinates. + + """ + from satpy.composites import CompositeBase + from satpy.resample import add_crs_xy_coords + + comp = CompositeBase("test_comp") + data_arr1 = self._get_test_ds(shape=(2, 2)) + data_arr1 = add_crs_xy_coords(data_arr1, data_arr1.attrs["area"]) + data_arr2 = self._get_test_ds(shape=(2, 2)) + data_arr2 = data_arr2.assign_coords( + x=data_arr1.coords["x"] + 0.000001, + y=data_arr1.coords["y"], + crs=data_arr1.coords["crs"], + ) + # data_arr2 = add_crs_xy_coords(data_arr2, data_arr2.attrs["area"]) + # data_arr2.assign_coords(x=data_arr2.coords["x"].copy() + 1.1) + # default xarray alignment would fail and collapse one of our dims + assert 0 in (data_arr2 - data_arr1).shape + new_data_arr1, new_data_arr2 = comp.match_data_arrays([data_arr1, data_arr2]) + assert 0 not in new_data_arr1.shape + assert 0 not in new_data_arr2.shape + assert 0 not in (new_data_arr2 - new_data_arr1).shape + class TestRatioSharpenedCompositors: """Test RatioSharpenedRGB and SelfSharpendRGB compositors.""" From cfbcaf76d8c726e0ffa1a7b7df9d2c81c3f4c4f2 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 14 Dec 2023 14:14:17 -0600 Subject: [PATCH 104/130] Cleanup match_data_arrays and add type annotations --- satpy/composites/__init__.py | 86 ++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index a0ce73caea..ef4a559322 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -157,7 +157,7 @@ def apply_modifier_info(self, origin, destination): elif o.get(k) is not None: d[k] = o[k] - def match_data_arrays(self, data_arrays): + def match_data_arrays(self, data_arrays: Sequence[xr.DataArray]) -> list[xr.DataArray]: """Match data arrays so that they can be used together in a composite. For the purpose of this method, "can be used together" means: @@ -189,46 +189,7 @@ def match_data_arrays(self, data_arrays): new_arrays = list(unify_chunks(*new_arrays)) return new_arrays - def drop_coordinates(self, data_arrays): - """Drop negligible non-dimensional coordinates. - - Drops negligible coordinates if they do not correspond to any - dimension. Negligible coordinates are defined in the - :attr:`NEGLIGIBLE_COORDS` module attribute. - - Args: - data_arrays (List[arrays]): Arrays to be checked - """ - new_arrays = [] - for ds in data_arrays: - drop = [coord for coord in ds.coords - if coord not in ds.dims and - any([neglible in coord for neglible in NEGLIGIBLE_COORDS])] - if drop: - new_arrays.append(ds.drop_vars(drop)) - else: - new_arrays.append(ds) - - return new_arrays - - def align_geo_coordinates(self, data_arrays: Sequence[xr.DataArray]) -> list[xr.DataArray]: - """Align DataArrays along geolocation coordinates. - - See :func:`~xarray.align` for more information. This function uses - the "override" join method to essentially ignore differences between - coordinates. The :meth:`check_geolocation` should be called before - this to ensure that geolocation coordinates and "area" are compatible. - The :meth:`drop_coordinates` method should be called before this to - ensure that coordinates that are considered "negligible" when computing - composites do not affect alignment. - - """ - non_geo_coords = tuple( - coord_name for data_arr in data_arrays - for coord_name in data_arr.coords if coord_name not in ("x", "y")) - return xr.align(*data_arrays, join="override", exclude=non_geo_coords) - - def check_geolocation(self, data_arrays): + def check_geolocation(self, data_arrays: Sequence[xr.DataArray]) -> None: """Check that the geolocations of the *data_arrays* are compatible. For the purpose of this method, "compatible" means: @@ -238,7 +199,7 @@ def check_geolocation(self, data_arrays): - If all have an area, the areas should be all the same. Args: - data_arrays (List[arrays]): Arrays to be checked + data_arrays: Arrays to be checked Raises: :class:`IncompatibleAreas`: @@ -269,6 +230,47 @@ def check_geolocation(self, data_arrays): "'{}'".format(self.attrs["name"])) raise IncompatibleAreas("Areas are different") + @staticmethod + def drop_coordinates(data_arrays: Sequence[xr.DataArray]) -> list[xr.DataArray]: + """Drop negligible non-dimensional coordinates. + + Drops negligible coordinates if they do not correspond to any + dimension. Negligible coordinates are defined in the + :attr:`NEGLIGIBLE_COORDS` module attribute. + + Args: + data_arrays (List[arrays]): Arrays to be checked + """ + new_arrays = [] + for ds in data_arrays: + drop = [coord for coord in ds.coords + if coord not in ds.dims and + any([neglible in coord for neglible in NEGLIGIBLE_COORDS])] + if drop: + new_arrays.append(ds.drop_vars(drop)) + else: + new_arrays.append(ds) + + return new_arrays + + @staticmethod + def align_geo_coordinates(data_arrays: Sequence[xr.DataArray]) -> list[xr.DataArray]: + """Align DataArrays along geolocation coordinates. + + See :func:`~xarray.align` for more information. This function uses + the "override" join method to essentially ignore differences between + coordinates. The :meth:`check_geolocation` should be called before + this to ensure that geolocation coordinates and "area" are compatible. + The :meth:`drop_coordinates` method should be called before this to + ensure that coordinates that are considered "negligible" when computing + composites do not affect alignment. + + """ + non_geo_coords = tuple( + coord_name for data_arr in data_arrays + for coord_name in data_arr.coords if coord_name not in ("x", "y")) + return list(xr.align(*data_arrays, join="override", exclude=non_geo_coords)) + class DifferenceCompositor(CompositeBase): """Make the difference of two data arrays.""" From fb8ff3ba2f14f5edf9a03b1edd1288943d3620f5 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Thu, 14 Dec 2023 21:39:21 +0100 Subject: [PATCH 105/130] import hvplot directly inside method As Martin has suggested I'm importing directly inside method the hvplot library to remove an if condition and resolve "too complex" pre-commit control --- satpy/scene.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index aea5b44cfe..fe6bbce1f9 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -39,12 +39,6 @@ from satpy.utils import convert_remote_files_to_fsspec, get_storage_options_from_reader_kwargs from satpy.writers import load_writer -#try: -# import hvplot.xarray as hvplot_xarray # noqa -#except ImportError: -# hvplot_xarray = None - - LOG = logging.getLogger(__name__) @@ -1074,12 +1068,12 @@ def to_geoviews(self, gvtype=None, datasets=None, kdims=None, vdims=None, dynami return gview def to_hvplot(self, datasets=None, *args, **kwargs): - """Convert satpy Scene to Hvplot. + """Convert satpy Scene to Hvplot. The method could not be used with composites of swath data. Args: datasets (list): Limit included products to these datasets. - kwargs: hvplot options dictionary. args: Arguments coming from hvplot + kwargs: hvplot options dictionary. Returns: hvplot object that contains within it the plots of datasets list. As default it contains all Scene datasets plots and a plot title is shown. @@ -1087,10 +1081,11 @@ def to_hvplot(self, datasets=None, *args, **kwargs): Example usage:: scene_list = ['ash','IR_108'] + scn = Scene() + scn.load(scene_list) + scn = scn.resample('eurol') plot = scn.to_hvplot(datasets=scene_list) - plot.ash+plot.IR_108 - """ def _get_crs(xarray_ds): @@ -1113,12 +1108,7 @@ def _plot_quadmesh(xarray_ds, variable, **defaults): clabel=f"[{_get_units(xarray_ds,variable)}]", title=title, **defaults) - #def _check_hvplot_library(): - # if hvplot_xarray is None: - # raise ImportError("'hvplot' must be installed to use this feature") -# -# _check_hvplot_library() - + import hvplot.xarray as hvplot_xarray # noqa plot = Overlay() xarray_ds = self.to_xarray_dataset(datasets) @@ -1129,7 +1119,6 @@ def _plot_quadmesh(xarray_ds, variable, **defaults): ccrs = None defaults={"x":"longitude","y":"latitude"} - if datasets is None: datasets = list(xarray_ds.keys()) @@ -1141,9 +1130,6 @@ def _plot_quadmesh(xarray_ds, variable, **defaults): defaults.update(kwargs) - #if "latitude" in xarray_ds.coords: - # defaults.update({"x":"longitude","y":"latitude"}) - for element in datasets: title = f"{element} @ {_get_timestamp(xarray_ds)}" if xarray_ds[element].shape[0] == 3: From 807357a4d4ea1a4f4cad740d5978d534b1e61b20 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Thu, 14 Dec 2023 22:45:58 +0100 Subject: [PATCH 106/130] Add holoviews required library --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 33d417bf2e..49b00ae4e0 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ "doc": ["sphinx", "sphinx_rtd_theme", "sphinxcontrib-apidoc"], # Other "geoviews": ["geoviews"], - "hvplot": ["hvplot", "geoviews", "cartopy"], + "hvplot": ["hvplot", "geoviews", "cartopy", "holoviews"], "overlays": ["pycoast", "pydecorate"], "satpos_from_tle": ["skyfield", "astropy"], "tests": test_requires, From 8b7bba7bede6bc1b0f3c8ebc4d70a5a2b8cea501 Mon Sep 17 00:00:00 2001 From: Johan Strandgren <42137969+strandgren@users.noreply.github.com> Date: Thu, 14 Dec 2023 23:41:10 +0100 Subject: [PATCH 107/130] Clean up tests for NDVIHybridGreen compositor. --- satpy/tests/compositor_tests/test_spectral.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/satpy/tests/compositor_tests/test_spectral.py b/satpy/tests/compositor_tests/test_spectral.py index 7472016c00..c9a7ab62b6 100644 --- a/satpy/tests/compositor_tests/test_spectral.py +++ b/satpy/tests/compositor_tests/test_spectral.py @@ -103,10 +103,10 @@ def test_ndvi_hybrid_green(self): # Test General functionality with linear strength (=1.0) res = comp((self.c01, self.c02, self.c03)) - assert isinstance(res, xr.DataArray) - assert isinstance(res.data, da.Array) - assert res.attrs["name"] == "ndvi_hybrid_green" - assert res.attrs["standard_name"] == "toa_bidirectional_reflectance" + assert isinstance(res, xr.DataArray) + assert isinstance(res.data, da.Array) + assert res.attrs["name"] == "ndvi_hybrid_green" + assert res.attrs["standard_name"] == "toa_bidirectional_reflectance" data = res.values np.testing.assert_array_almost_equal(data, np.array([[0.2633, 0.3071], [0.2115, 0.3420]]), decimal=4) @@ -115,7 +115,7 @@ def test_ndvi_hybrid_green_dtype(self): with dask.config.set(scheduler=CustomScheduler(max_computes=0)): comp = NDVIHybridGreen("ndvi_hybrid_green", limits=(0.15, 0.05), prerequisites=(0.51, 0.65, 0.85), standard_name="toa_bidirectional_reflectance") - res = comp((self.c01, self.c02, self.c03)).compute() + res = comp((self.c01, self.c02, self.c03)) assert res.data.dtype == np.float32 def test_nonlinear_scaling(self): @@ -124,7 +124,6 @@ def test_nonlinear_scaling(self): comp = NDVIHybridGreen("ndvi_hybrid_green", limits=(0.15, 0.05), strength=2.0, prerequisites=(0.51, 0.65, 0.85), standard_name="toa_bidirectional_reflectance") - res = comp((self.c01, self.c02, self.c03)) res_np = res.data.compute() assert res.dtype == res_np.dtype From ebceffbd015fcf6c31056a328316f9d28669674c Mon Sep 17 00:00:00 2001 From: Johan Strandgren <42137969+strandgren@users.noreply.github.com> Date: Thu, 14 Dec 2023 23:44:09 +0100 Subject: [PATCH 108/130] Add instance checks. --- satpy/tests/test_composites.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index aa4f56f9de..d17188846c 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -969,6 +969,8 @@ def test_high_cloud_compositor(self): with dask.config.set(scheduler=CustomScheduler(max_computes=0)): comp = HighCloudCompositor(name="test") res = comp([self.data]) + assert isinstance(res, xr.DataArray) + assert isinstance(res.data, da.Array) expexted_alpha = np.array([[1.0, 0.7142857, 0.0], [1.0, 0.625, 0.0], [1.0, 0.5555555, 0.0]]) expected = np.stack([self.data, expexted_alpha]) np.testing.assert_almost_equal(res.values, expected) @@ -1014,6 +1016,8 @@ def test_low_cloud_compositor(self): with dask.config.set(scheduler=CustomScheduler(max_computes=0)): comp = LowCloudCompositor(name="test") res = comp([self.btd, self.bt_win, self.lsm]) + assert isinstance(res, xr.DataArray) + assert isinstance(res.data, da.Array) expexted_alpha = np.array([[0.0, 0.25, 1.0], [0.0, 0.25, 1.0], [0.0, 0.0, 0.0]]) expected = np.stack([self.btd, expexted_alpha]) np.testing.assert_equal(res.values, expected) From 9c277391fc20a7883533a3441fea17de72f23a1d Mon Sep 17 00:00:00 2001 From: Johan Strandgren <42137969+strandgren@users.noreply.github.com> Date: Thu, 14 Dec 2023 23:49:30 +0100 Subject: [PATCH 109/130] Remove trailing whitespace. --- satpy/tests/test_composites.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index d17188846c..4f82467ab9 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -970,7 +970,7 @@ def test_high_cloud_compositor(self): comp = HighCloudCompositor(name="test") res = comp([self.data]) assert isinstance(res, xr.DataArray) - assert isinstance(res.data, da.Array) + assert isinstance(res.data, da.Array) expexted_alpha = np.array([[1.0, 0.7142857, 0.0], [1.0, 0.625, 0.0], [1.0, 0.5555555, 0.0]]) expected = np.stack([self.data, expexted_alpha]) np.testing.assert_almost_equal(res.values, expected) From aad6ea810f237ae481ed72c63357a0bc7f532bf8 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Thu, 14 Dec 2023 22:45:58 +0100 Subject: [PATCH 110/130] Add holoviews required library --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 33d417bf2e..d31c21364a 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,8 @@ "doc": ["sphinx", "sphinx_rtd_theme", "sphinxcontrib-apidoc"], # Other "geoviews": ["geoviews"], - "hvplot": ["hvplot", "geoviews", "cartopy"], + "hvplot": ["hvplot", "geoviews", "cartopy", "holoviews"], + "holoviews": ["holoviews"], "overlays": ["pycoast", "pydecorate"], "satpos_from_tle": ["skyfield", "astropy"], "tests": test_requires, From fca35cd5772335093b1e2defdbf7a52fb7f804da Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Thu, 14 Dec 2023 22:45:58 +0100 Subject: [PATCH 111/130] Add holoviews required library --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index d31c21364a..49b00ae4e0 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,6 @@ # Other "geoviews": ["geoviews"], "hvplot": ["hvplot", "geoviews", "cartopy", "holoviews"], - "holoviews": ["holoviews"], "overlays": ["pycoast", "pydecorate"], "satpos_from_tle": ["skyfield", "astropy"], "tests": test_requires, From 7918f375de45272055b26d8d7ba1f3caf4aba759 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 15 Dec 2023 00:44:30 +0100 Subject: [PATCH 112/130] Revert "Add holoviews required library" This reverts commit 807357a4d4ea1a4f4cad740d5978d534b1e61b20. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 49b00ae4e0..33d417bf2e 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ "doc": ["sphinx", "sphinx_rtd_theme", "sphinxcontrib-apidoc"], # Other "geoviews": ["geoviews"], - "hvplot": ["hvplot", "geoviews", "cartopy", "holoviews"], + "hvplot": ["hvplot", "geoviews", "cartopy"], "overlays": ["pycoast", "pydecorate"], "satpos_from_tle": ["skyfield", "astropy"], "tests": test_requires, From 1e9dbf29d90ba6c72bfa3110e1006b76baf66b8c Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Fri, 15 Dec 2023 06:25:20 +0000 Subject: [PATCH 113/130] Add data validity tests. --- satpy/composites/__init__.py | 4 ++-- satpy/tests/test_composites.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 4153a85963..a286f078c4 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1100,8 +1100,8 @@ def __init__(self, name, transition_min_limits=(210., 230.), latitude_min_limits """ if len(transition_min_limits) != 2: raise ValueError(f"Expected 2 `transition_min_limits` values, got {len(transition_min_limits)}") - if len(transition_min_limits) != 2: - raise ValueError(f"Expected 2 `latitude_min_limits` values, got {len(transition_min_limits)}") + if len(latitude_min_limits) != 2: + raise ValueError(f"Expected 2 `latitude_min_limits` values, got {len(latitude_min_limits)}") if type(transition_max) in [list, tuple]: raise ValueError(f"Expected `transition_max` to be of type float, is of type {type(transition_max)}") diff --git a/satpy/tests/test_composites.py b/satpy/tests/test_composites.py index 4f82467ab9..f6726bc7e9 100644 --- a/satpy/tests/test_composites.py +++ b/satpy/tests/test_composites.py @@ -990,6 +990,24 @@ def test_high_cloud_compositor_dtype(self): res = comp([self.data]) assert res.data.dtype == self.dtype + def test_high_cloud_compositor_validity_checks(self): + """Test that errors are raised for invalid input data and settings.""" + from satpy.composites import HighCloudCompositor + + with pytest.raises(ValueError, match="Expected 2 `transition_min_limits` values, got 1"): + _ = HighCloudCompositor("test", transition_min_limits=(210., )) + + with pytest.raises(ValueError, match="Expected 2 `latitude_min_limits` values, got 3"): + _ = HighCloudCompositor("test", latitude_min_limits=(20., 40., 60.)) + + with pytest.raises(ValueError, match="Expected `transition_max` to be of type float, " + "is of type "): + _ = HighCloudCompositor("test", transition_max=(250., 300.)) + + comp = HighCloudCompositor("test") + with pytest.raises(ValueError, match="Expected 1 dataset, got 2"): + _ = comp([self.data, self.data]) + class TestLowCloudCompositor: """Test LowCloudCompositor.""" @@ -1029,6 +1047,20 @@ def test_low_cloud_compositor_dtype(self): res = comp([self.btd, self.bt_win, self.lsm]) assert res.data.dtype == self.dtype + def test_low_cloud_compositor_validity_checks(self): + """Test that errors are raised for invalid input data and settings.""" + from satpy.composites import LowCloudCompositor + + with pytest.raises(ValueError, match="Expected 2 `range_land` values, got 1"): + _ = LowCloudCompositor("test", range_land=(2.0, )) + + with pytest.raises(ValueError, match="Expected 2 `range_water` values, got 1"): + _ = LowCloudCompositor("test", range_water=(2.0,)) + + comp = LowCloudCompositor("test") + with pytest.raises(ValueError, match="Expected 3 datasets, got 2"): + _ = comp([self.btd, self.lsm]) + class TestSingleBandCompositor(unittest.TestCase): """Test the single-band compositor.""" From 30eaf9ed65d6ba89948a1bcbad67099e676032b5 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Fri, 15 Dec 2023 06:26:33 +0000 Subject: [PATCH 114/130] Rename 'sea' to 'water'. --- satpy/composites/__init__.py | 34 +++++++++++++++++----------------- satpy/etc/composites/abi.yaml | 4 ++-- satpy/etc/composites/ahi.yaml | 4 ++-- satpy/etc/composites/fci.yaml | 6 +++--- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index a286f078c4..88375bc933 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1148,8 +1148,8 @@ class LowCloudCompositor(CloudCompositor): Pixels with `BTD` values below a given threshold will be transparent, whereas pixels with `BTD` values above another threshold will be opaque. The transparency of all other `BTD` values will be a linear function of the `BTD` value itself. Two sets of thresholds are used, one set for land surface types - (`range_land`) and another one for sea/water surface types (`range_sea`), respectively. Hence, - this compositor requires a land-sea-mask as a prerequisite input. This follows the GeoColor + (`range_land`) and another one for water surface types (`range_water`), respectively. Hence, + this compositor requires a land-water-mask as a prerequisite input. This follows the GeoColor implementation of night-time low-level clouds in Miller et al. (2020, :doi:`10.1175/JTECH-D-19-0134.1`), but with some adjustments to the thresholds based on recent developments and feedback from CIRA. @@ -1157,9 +1157,9 @@ class LowCloudCompositor(CloudCompositor): only applicable during night-time. """ - def __init__(self, name, values_land=(1,), values_sea=(0,), # noqa: D417 + def __init__(self, name, values_land=(1,), values_water=(0,), # noqa: D417 range_land=(0.0, 4.0), - range_sea=(0.0, 4.0), + range_water=(0.0, 4.0), invert_alpha=True, transition_gamma=1.0, **kwargs): """Init info. @@ -1167,12 +1167,12 @@ def __init__(self, name, values_land=(1,), values_sea=(0,), # noqa: D417 Collect custom configuration values. Args: - values_land (list): List of values used to identify land surface pixels in the land-sea-mask. - values_sea (list): List of values used to identify sea/water surface pixels in the land-sea-mask. + values_land (list): List of values used to identify land surface pixels in the land-water-mask. + values_water (list): List of values used to identify water surface pixels in the land-water-mask. range_land (tuple): Threshold values used for masking low-level clouds from the brightness temperature difference over land surface types. - range_sea (tuple): Threshold values used for masking low-level clouds from the brightness temperature - difference over sea/water. + range_water (tuple): Threshold values used for masking low-level clouds from the brightness temperature + difference over water. invert_alpha (bool): Invert the alpha channel to make low data values transparent and high data values opaque. transition_gamma (float): Gamma correction to apply to the alpha channel within the brightness @@ -1180,13 +1180,13 @@ def __init__(self, name, values_land=(1,), values_sea=(0,), # noqa: D417 """ if len(range_land) != 2: raise ValueError(f"Expected 2 `range_land` values, got {len(range_land)}") - if len(range_sea) != 2: - raise ValueError(f"Expected 2 `range_sea` values, got {len(range_sea)}") + if len(range_water) != 2: + raise ValueError(f"Expected 2 `range_water` values, got {len(range_water)}") self.values_land = values_land if type(values_land) in [list, tuple] else [values_land] - self.values_sea = values_sea if type(values_sea) in [list, tuple] else [values_sea] + self.values_water = values_water if type(values_water) in [list, tuple] else [values_water] self.range_land = range_land - self.range_sea = range_sea + self.range_water = range_water super().__init__(name, transition_min=None, transition_max=None, invert_alpha=invert_alpha, transition_gamma=transition_gamma, **kwargs) @@ -1211,12 +1211,12 @@ def __call__(self, projectables, **kwargs): self.transition_min, self.transition_max = self.range_land res = super().__call__([btd.where(lsm.isin(self.values_land))], **kwargs) - # Call CloudCompositor for sea/water surface pixels - self.transition_min, self.transition_max = self.range_sea - res_sea = super().__call__([btd.where(lsm.isin(self.values_sea))], **kwargs) + # Call CloudCompositor for /water surface pixels + self.transition_min, self.transition_max = self.range_water + res_water = super().__call__([btd.where(lsm.isin(self.values_water))], **kwargs) - # Compine resutls for land and sea/water surface pixels - res = res.where(lsm.isin(self.values_land), res_sea) + # Compine resutls for land and water surface pixels + res = res.where(lsm.isin(self.values_land), res_water) # Make pixels with cold window channel brightness temperatures transparent to avoid spurious false # alarms caused by noise in the 3.9um channel that can occur for very cold cloud tops diff --git a/satpy/etc/composites/abi.yaml b/satpy/etc/composites/abi.yaml index b40f353e6a..cae7a7035f 100644 --- a/satpy/etc/composites/abi.yaml +++ b/satpy/etc/composites/abi.yaml @@ -773,7 +773,7 @@ composites: geo_color_low_clouds: standard_name: geo_color_low_clouds compositor: !!python/name:satpy.composites.LowCloudCompositor - values_sea: 0 + values_water: 0 values_land: 100 prerequisites: - compositor: !!python/name:satpy.composites.DifferenceCompositor @@ -782,7 +782,7 @@ composites: - name: C07 - name: C13 - compositor: !!python/name:satpy.composites.StaticImageCompositor - standard_name: land_sea_mask + standard_name: land_water_mask url: "https://zenodo.org/records/10076199/files/gshhs_land_water_mask_3km_i.tif" known_hash: "sha256:96df83c57416217e191f95dde3d3c1ce0373a8fc220e929228873db246ca3569" diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 066b9cf0f2..3db0d20f3c 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -523,7 +523,7 @@ composites: geo_color_low_clouds: standard_name: geo_color_low_clouds compositor: !!python/name:satpy.composites.LowCloudCompositor - values_sea: 0 + values_water: 0 values_land: 100 prerequisites: - compositor: !!python/name:satpy.composites.DifferenceCompositor @@ -532,7 +532,7 @@ composites: - name: B07 - name: B13 - compositor: !!python/name:satpy.composites.StaticImageCompositor - standard_name: land_sea_mask + standard_name: land_water_mask url: "https://zenodo.org/records/10076199/files/gshhs_land_water_mask_3km_i.tif" known_hash: "sha256:96df83c57416217e191f95dde3d3c1ce0373a8fc220e929228873db246ca3569" diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index f27011d301..775331c08b 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -145,10 +145,10 @@ composites: geo_color_low_clouds: standard_name: geo_color_low_clouds compositor: !!python/name:satpy.composites.LowCloudCompositor - values_sea: 0 + values_water: 0 values_land: 100 + range_water: [1.35, 5.0] range_land: [4.35, 6.75] - range_sea: [1.35, 5.0] prerequisites: - compositor: !!python/name:satpy.composites.DifferenceCompositor prerequisites: @@ -156,7 +156,7 @@ composites: - name: ir_38 - name: ir_105 - compositor: !!python/name:satpy.composites.StaticImageCompositor - standard_name: land_sea_mask + standard_name: land_water_mask url: "https://zenodo.org/records/10076199/files/gshhs_land_water_mask_3km_i.tif" known_hash: "sha256:96df83c57416217e191f95dde3d3c1ce0373a8fc220e929228873db246ca3569" From 60fa8a1c9d914f223a4917de63baff191ffa9f8a Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Fri, 15 Dec 2023 06:37:21 +0000 Subject: [PATCH 115/130] Add description and reference in geo_color composite recipes. --- satpy/etc/composites/abi.yaml | 7 +++++++ satpy/etc/composites/ahi.yaml | 8 +++++++- satpy/etc/composites/fci.yaml | 7 +++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/satpy/etc/composites/abi.yaml b/satpy/etc/composites/abi.yaml index cae7a7035f..4700aa470b 100644 --- a/satpy/etc/composites/abi.yaml +++ b/satpy/etc/composites/abi.yaml @@ -756,6 +756,13 @@ composites: # GeoColor geo_color: compositor: !!python/name:satpy.composites.DayNightCompositor + description: > + GeoColor is a multi-layer blended RGB composite where the day-time part of the image is represented by true + color imagery and the nighttime part of the image by a three layer vertically blended stack composed of a + high-level cloud layer (single IR window channel), a low-level cloud layer (IR split window) and a static + surface terrain layer with city lights (NASA Black Marble). + references: + Research Article: https://journals.ametsoc.org/view/journals/atot/37/3/JTECH-D-19-0134.1.xml lim_low: 78 lim_high: 88 standard_name: geo_color_day_night_blend diff --git a/satpy/etc/composites/ahi.yaml b/satpy/etc/composites/ahi.yaml index 3db0d20f3c..fe64a2bcae 100644 --- a/satpy/etc/composites/ahi.yaml +++ b/satpy/etc/composites/ahi.yaml @@ -503,9 +503,15 @@ composites: - _night_background_hires # GeoColor - geo_color: compositor: !!python/name:satpy.composites.DayNightCompositor + description: > + GeoColor is a multi-layer blended RGB composite where the day-time part of the image is represented by true + color imagery and the nighttime part of the image by a three layer vertically blended stack composed of a + high-level cloud layer (single IR window channel), a low-level cloud layer (IR split window) and a static + surface terrain layer with city lights (NASA Black Marble). + references: + Research Article: https://journals.ametsoc.org/view/journals/atot/37/3/JTECH-D-19-0134.1.xml lim_low: 78 lim_high: 88 standard_name: geo_color_day_night_blend diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 775331c08b..0f0e98f4e0 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -128,6 +128,13 @@ composites: # GeoColor geo_color: compositor: !!python/name:satpy.composites.DayNightCompositor + description: > + GeoColor is a multi-layer blended RGB composite where the day-time part of the image is represented by true + color imagery and the nighttime part of the image by a three layer vertically blended stack composed of a + high-level cloud layer (single IR window channel), a low-level cloud layer (IR split window) and a static + surface terrain layer with city lights (NASA Black Marble). + references: + Research Article: https://journals.ametsoc.org/view/journals/atot/37/3/JTECH-D-19-0134.1.xml lim_low: 78 lim_high: 88 standard_name: geo_color_day_night_blend From 4df285d66c1882075e30ea0f064808098be17af9 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Dec 2023 09:05:01 +0200 Subject: [PATCH 116/130] Fix proj authority usage --- satpy/tests/writer_tests/test_mitiff.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index b4ff371dab..f1519cf7a1 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -844,23 +844,23 @@ def test_convert_proj4_string(self): from pyresample.geometry import AreaDefinition from satpy.writers.mitiff import MITIFFWriter - checks = [{"epsg": "+init=EPSG:32631", + checks = [{"epsg": "EPSG:32631", "proj4": (" Proj string: +proj=etmerc +lat_0=0 +lon_0=3 +k=0.9996 " "+ellps=WGS84 +datum=WGS84 +units=km +x_0=501020.000000 " "+y_0=1515.000000\n")}, - {"epsg": "+init=EPSG:32632", + {"epsg": "EPSG:32632", "proj4": (" Proj string: +proj=etmerc +lat_0=0 +lon_0=9 +k=0.9996 " "+ellps=WGS84 +datum=WGS84 +units=km +x_0=501020.000000 " "+y_0=1515.000000\n")}, - {"epsg": "+init=EPSG:32633", + {"epsg": "EPSG:32633", "proj4": (" Proj string: +proj=etmerc +lat_0=0 +lon_0=15 +k=0.9996 " "+ellps=WGS84 +datum=WGS84 +units=km +x_0=501020.000000 " "+y_0=1515.000000\n")}, - {"epsg": "+init=EPSG:32634", + {"epsg": "EPSG:32634", "proj4": (" Proj string: +proj=etmerc +lat_0=0 +lon_0=21 +k=0.9996 " "+ellps=WGS84 +datum=WGS84 +units=km +x_0=501020.000000 " "+y_0=1515.000000\n")}, - {"epsg": "+init=EPSG:32635", + {"epsg": "EPSG:32635", "proj4": (" Proj string: +proj=etmerc +lat_0=0 +lon_0=27 +k=0.9996 " "+ellps=WGS84 +datum=WGS84 +units=km +x_0=501020.000000 " "+y_0=1515.000000\n")}] From 013b49fe2e014d39995bd05f9372e60f1812acf0 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Dec 2023 09:19:31 +0200 Subject: [PATCH 117/130] Fix/suppress PROJ warnings about losing projection information --- satpy/tests/writer_tests/test_mitiff.py | 54 +++++++++++-------------- satpy/writers/mitiff.py | 8 +++- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/satpy/tests/writer_tests/test_mitiff.py b/satpy/tests/writer_tests/test_mitiff.py index f1519cf7a1..4e8878687a 100644 --- a/satpy/tests/writer_tests/test_mitiff.py +++ b/satpy/tests/writer_tests/test_mitiff.py @@ -52,14 +52,13 @@ def _get_test_datasets(self): import dask.array as da import xarray as xr + from pyproj import CRS from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict area_def = AreaDefinition( "test", "test", "test", - proj4_str_to_dict("+proj=stere +datum=WGS84 +ellps=WGS84 " - "+lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), + CRS("+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), 100, 200, (-1000., -1500., 1000., 1500.), @@ -119,14 +118,13 @@ def _get_test_datasets_sensor_set(self): import dask.array as da import xarray as xr + from pyproj import CRS from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict area_def = AreaDefinition( "test", "test", "test", - proj4_str_to_dict("+proj=stere +datum=WGS84 +ellps=WGS84 " - "+lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), + CRS("+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), 100, 200, (-1000., -1500., 1000., 1500.), @@ -186,14 +184,14 @@ def _get_test_dataset(self, bands=3): import dask.array as da import xarray as xr + from pyproj import CRS from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict + area_def = AreaDefinition( "test", "test", "test", - proj4_str_to_dict("+proj=stere +datum=WGS84 +ellps=WGS84 " - "+lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), + CRS("+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), 100, 200, (-1000., -1500., 1000., 1500.), @@ -217,14 +215,14 @@ def _get_test_one_dataset(self): import dask.array as da import xarray as xr + from pyproj import CRS from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict + area_def = AreaDefinition( "test", "test", "test", - proj4_str_to_dict("+proj=geos +datum=WGS84 +ellps=WGS84 " - "+lon_0=0. h=36000. +units=km"), + CRS("+proj=geos +datum=WGS84 +ellps=WGS84 +lon_0=0. h=36000. +units=km"), 100, 200, (-1000., -1500., 1000., 1500.), @@ -248,14 +246,14 @@ def _get_test_one_dataset_sensor_set(self): import dask.array as da import xarray as xr + from pyproj import CRS from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict + area_def = AreaDefinition( "test", "test", "test", - proj4_str_to_dict("+proj=geos +datum=WGS84 +ellps=WGS84 " - "+lon_0=0. h=36000. +units=km"), + CRS("+proj=geos +datum=WGS84 +ellps=WGS84 +lon_0=0. h=36000. +units=km"), 100, 200, (-1000., -1500., 1000., 1500.), @@ -278,14 +276,14 @@ def _get_test_dataset_with_bad_values(self, bands=3): from datetime import datetime import xarray as xr + from pyproj import CRS from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict + area_def = AreaDefinition( "test", "test", "test", - proj4_str_to_dict("+proj=stere +datum=WGS84 +ellps=WGS84 " - "+lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), + CRS("+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), 100, 200, (-1000., -1500., 1000., 1500.), @@ -313,8 +311,8 @@ def _get_test_dataset_calibration(self, bands=6): import dask.array as da import xarray as xr + from pyproj import CRS from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict from satpy.scene import Scene from satpy.tests.utils import make_dsq @@ -322,8 +320,7 @@ def _get_test_dataset_calibration(self, bands=6): "test", "test", "test", - proj4_str_to_dict("+proj=stere +datum=WGS84 +ellps=WGS84 " - "+lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), + CRS("+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), 100, 200, (-1000., -1500., 1000., 1500.), @@ -418,8 +415,8 @@ def _get_test_dataset_calibration_one_dataset(self, bands=1): import dask.array as da import xarray as xr + from pyproj import CRS from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict from satpy.scene import Scene from satpy.tests.utils import make_dsq @@ -427,8 +424,7 @@ def _get_test_dataset_calibration_one_dataset(self, bands=1): "test", "test", "test", - proj4_str_to_dict("+proj=stere +datum=WGS84 +ellps=WGS84 " - "+lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), + CRS("+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), 100, 200, (-1000., -1500., 1000., 1500.), @@ -473,16 +469,15 @@ def _get_test_dataset_three_bands_two_prereq(self, bands=3): import dask.array as da import xarray as xr + from pyproj import CRS from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict from satpy.tests.utils import make_dsq area_def = AreaDefinition( "test", "test", "test", - proj4_str_to_dict("+proj=stere +datum=WGS84 +ellps=WGS84 " - "+lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), + CRS("+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), 100, 200, (-1000., -1500., 1000., 1500.), @@ -508,16 +503,15 @@ def _get_test_dataset_three_bands_prereq(self, bands=3): import dask.array as da import xarray as xr + from pyproj import CRS from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict from satpy.tests.utils import make_dsq area_def = AreaDefinition( "test", "test", "test", - proj4_str_to_dict("+proj=stere +datum=WGS84 +ellps=WGS84 " - "+lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), + CRS("+proj=stere +datum=WGS84 +ellps=WGS84 +lon_0=0. +lat_0=90 +lat_ts=60 +units=km"), 100, 200, (-1000., -1500., 1000., 1500.), diff --git a/satpy/writers/mitiff.py b/satpy/writers/mitiff.py index 950fce8b21..3658ac16b7 100644 --- a/satpy/writers/mitiff.py +++ b/satpy/writers/mitiff.py @@ -221,6 +221,8 @@ def _add_sizes(self, datasets, first_dataset): return _image_description def _add_proj4_string(self, datasets, first_dataset): + import warnings + proj4_string = " Proj string: " if isinstance(datasets, list): @@ -232,7 +234,11 @@ def _add_proj4_string(self, datasets, first_dataset): if hasattr(area, "crs") and area.crs.to_epsg() is not None: proj4_string += "+init=EPSG:{}".format(area.crs.to_epsg()) else: - proj4_string += area.proj_str + # Filter out the PROJ warning of losing projection information + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning, + message=r"You will likely lose important projection information") + proj4_string += area.proj_str x_0 = 0 y_0 = 0 From 755ec6b886ac2516e5c0f8c8021f9637d75a39b7 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Dec 2023 09:38:22 +0200 Subject: [PATCH 118/130] Use datetime64[ns] in CF writer --- satpy/cf/coords.py | 4 ++-- satpy/tests/writer_tests/test_cf.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/satpy/cf/coords.py b/satpy/cf/coords.py index 9220632fcb..2449ab79ee 100644 --- a/satpy/cf/coords.py +++ b/satpy/cf/coords.py @@ -291,8 +291,8 @@ def add_time_bounds_dimension(ds: xr.Dataset, time: str = "time") -> xr.Dataset: if start_time is not None) end_time = min(end_time for end_time in end_times if end_time is not None) - ds["time_bnds"] = xr.DataArray([[np.datetime64(start_time), - np.datetime64(end_time)]], + ds["time_bnds"] = xr.DataArray([[np.datetime64(start_time, "ns"), + np.datetime64(end_time, "ns")]], dims=["time", "bnds_1d"]) ds[time].attrs["bounds"] = "time_bnds" ds[time].attrs["standard_name"] = "time" diff --git a/satpy/tests/writer_tests/test_cf.py b/satpy/tests/writer_tests/test_cf.py index 62c9995cde..d37b612bb2 100644 --- a/satpy/tests/writer_tests/test_cf.py +++ b/satpy/tests/writer_tests/test_cf.py @@ -240,7 +240,7 @@ def test_single_time_value(self): test_array = np.array([[1, 2], [3, 4]]) scn["test-array"] = xr.DataArray(test_array, dims=["x", "y"], - coords={"time": np.datetime64("2018-05-30T10:05:00")}, + coords={"time": np.datetime64("2018-05-30T10:05:00", "ns")}, attrs=dict(start_time=start_time, end_time=end_time)) with TempFile() as filename: @@ -255,7 +255,7 @@ def test_time_coordinate_on_a_swath(self): scn = Scene() test_array = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) times = np.array(["2018-05-30T10:05:00", "2018-05-30T10:05:01", - "2018-05-30T10:05:02", "2018-05-30T10:05:03"], dtype=np.datetime64) + "2018-05-30T10:05:02", "2018-05-30T10:05:03"], dtype="datetime64[ns]") scn["test-array"] = xr.DataArray(test_array, dims=["y", "x"], coords={"time": ("y", times)}, @@ -273,7 +273,7 @@ def test_bounds(self): test_array = np.array([[1, 2], [3, 4]]).reshape(2, 2, 1) scn["test-array"] = xr.DataArray(test_array, dims=["x", "y", "time"], - coords={"time": [np.datetime64("2018-05-30T10:05:00")]}, + coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]}, attrs=dict(start_time=start_time, end_time=end_time)) with TempFile() as filename: @@ -307,12 +307,12 @@ def test_bounds_minimum(self): test_arrayB = np.array([[1, 2], [3, 5]]).reshape(2, 2, 1) scn["test-arrayA"] = xr.DataArray(test_arrayA, dims=["x", "y", "time"], - coords={"time": [np.datetime64("2018-05-30T10:05:00")]}, + coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]}, attrs=dict(start_time=start_timeA, end_time=end_timeA)) scn["test-arrayB"] = xr.DataArray(test_arrayB, dims=["x", "y", "time"], - coords={"time": [np.datetime64("2018-05-30T10:05:00")]}, + coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]}, attrs=dict(start_time=start_timeB, end_time=end_timeB)) with TempFile() as filename: @@ -330,12 +330,12 @@ def test_bounds_missing_time_info(self): test_arrayB = np.array([[1, 2], [3, 5]]).reshape(2, 2, 1) scn["test-arrayA"] = xr.DataArray(test_arrayA, dims=["x", "y", "time"], - coords={"time": [np.datetime64("2018-05-30T10:05:00")]}, + coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]}, attrs=dict(start_time=start_timeA, end_time=end_timeA)) scn["test-arrayB"] = xr.DataArray(test_arrayB, dims=["x", "y", "time"], - coords={"time": [np.datetime64("2018-05-30T10:05:00")]}) + coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]}) with TempFile() as filename: scn.save_datasets(filename=filename, writer="cf") with xr.open_dataset(filename, decode_cf=True) as f: @@ -350,7 +350,7 @@ def test_unlimited_dims_kwarg(self): test_array = np.array([[1, 2], [3, 4]]) scn["test-array"] = xr.DataArray(test_array, dims=["x", "y"], - coords={"time": np.datetime64("2018-05-30T10:05:00")}, + coords={"time": np.datetime64("2018-05-30T10:05:00", "ns")}, attrs=dict(start_time=start_time, end_time=end_time)) with TempFile() as filename: From cd4cd7362bc3f62b0129f7cb466e031f04bc270e Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Dec 2023 09:57:42 +0200 Subject: [PATCH 119/130] Catch warning about pretty time formatting --- satpy/tests/writer_tests/test_cf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/satpy/tests/writer_tests/test_cf.py b/satpy/tests/writer_tests/test_cf.py index d37b612bb2..bb87ff8c30 100644 --- a/satpy/tests/writer_tests/test_cf.py +++ b/satpy/tests/writer_tests/test_cf.py @@ -208,8 +208,10 @@ def test_groups(self): attrs={"name": "HRV", "start_time": tstart, "end_time": tend}) with TempFile() as filename: - scn.save_datasets(filename=filename, writer="cf", groups={"visir": ["IR_108", "VIS006"], "hrv": ["HRV"]}, - pretty=True) + with pytest.warns(UserWarning, match=r"Cannot pretty-format"): + scn.save_datasets(filename=filename, writer="cf", + groups={"visir": ["IR_108", "VIS006"], "hrv": ["HRV"]}, + pretty=True) nc_root = xr.open_dataset(filename) assert "history" in nc_root.attrs From 36b09d1f0036a6f47927d0a819ed9d38bb0c113e Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Dec 2023 10:00:27 +0200 Subject: [PATCH 120/130] Catch warning of invalid NetCDF dataset name --- satpy/tests/writer_tests/test_cf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/satpy/tests/writer_tests/test_cf.py b/satpy/tests/writer_tests/test_cf.py index bb87ff8c30..020cb10ec3 100644 --- a/satpy/tests/writer_tests/test_cf.py +++ b/satpy/tests/writer_tests/test_cf.py @@ -152,7 +152,8 @@ def test_save_dataset_a_digit_no_prefix_include_attr(self): scn = Scene() scn["1"] = xr.DataArray([1, 2, 3]) with TempFile() as filename: - scn.save_datasets(filename=filename, writer="cf", include_orig_name=True, numeric_name_prefix="") + with pytest.warns(UserWarning, match=r"Invalid NetCDF dataset name"): + scn.save_datasets(filename=filename, writer="cf", include_orig_name=True, numeric_name_prefix="") with xr.open_dataset(filename) as f: np.testing.assert_array_equal(f["1"][:], [1, 2, 3]) assert "original_name" not in f["1"].attrs From 8ea9e300bfdd48eff704cdbe68072e2442e3e45f Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Dec 2023 10:14:53 +0200 Subject: [PATCH 121/130] Add area definitions to remove unnecessary warnings --- satpy/tests/test_writers.py | 16 +++++++++++++++- satpy/tests/writer_tests/test_geotiff.py | 18 ++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/satpy/tests/test_writers.py b/satpy/tests/test_writers.py index e2bfd898ab..bc68d767c1 100644 --- a/satpy/tests/test_writers.py +++ b/satpy/tests/test_writers.py @@ -548,13 +548,20 @@ def setUp(self): import tempfile from datetime import datetime + from pyresample.geometry import AreaDefinition + from satpy.scene import Scene + adef = AreaDefinition( + "test", "test", "test", "EPSG:4326", + 100, 200, (-180., -90., 180., 90.), + ) ds1 = xr.DataArray( da.zeros((100, 200), chunks=50), dims=("y", "x"), attrs={"name": "test", - "start_time": datetime(2018, 1, 1, 0, 0, 0)} + "start_time": datetime(2018, 1, 1, 0, 0, 0), + "area": adef} ) self.scn = Scene() self.scn["test"] = ds1 @@ -650,8 +657,14 @@ def setup_method(self): import tempfile from datetime import datetime + from pyresample.geometry import AreaDefinition + from satpy.scene import Scene + adef = AreaDefinition( + "test", "test", "test", "EPSG:4326", + 100, 200, (-180., -90., 180., 90.), + ) ds1 = xr.DataArray( da.zeros((100, 200), chunks=50), dims=("y", "x"), @@ -659,6 +672,7 @@ def setup_method(self): "name": "test", "start_time": datetime(2018, 1, 1, 0, 0, 0), "sensor": "fake_sensor", + "area": adef, } ) ds2 = ds1.copy() diff --git a/satpy/tests/writer_tests/test_geotiff.py b/satpy/tests/writer_tests/test_geotiff.py index 74fcd43609..8925857637 100644 --- a/satpy/tests/writer_tests/test_geotiff.py +++ b/satpy/tests/writer_tests/test_geotiff.py @@ -32,12 +32,19 @@ def _get_test_datasets_2d(): """Create a single 2D test dataset.""" + from pyresample.geometry import AreaDefinition + + adef = AreaDefinition( + "test", "test", "test", "EPSG:4326", + 100, 200, (-180., -90., 180., 90.), + ) ds1 = xr.DataArray( da.zeros((100, 200), chunks=50), dims=("y", "x"), attrs={"name": "test", "start_time": datetime.utcnow(), - "units": "K"} + "units": "K", + "area": adef} ) return [ds1] @@ -54,12 +61,19 @@ def _get_test_datasets_2d_nonlinear_enhancement(): def _get_test_datasets_3d(): """Create a single 3D test dataset.""" + from pyresample.geometry import AreaDefinition + + adef = AreaDefinition( + "test", "test", "test", "EPSG:4326", + 100, 200, (-180., -90., 180., 90.), + ) ds1 = xr.DataArray( da.zeros((3, 100, 200), chunks=50), dims=("bands", "y", "x"), coords={"bands": ["R", "G", "B"]}, attrs={"name": "test", - "start_time": datetime.utcnow()} + "start_time": datetime.utcnow(), + "area": adef} ) return [ds1] From d16728b5d2a9c24aa14d263bc58214fa63ad47b2 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 15 Dec 2023 09:14:55 +0100 Subject: [PATCH 122/130] Add holoviews in documentation --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 3aa810420e..37c197c6eb 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -70,7 +70,7 @@ def __getattr__(cls, name): for mod_name in MOCK_MODULES: sys.modules[mod_name] = Mock() # type: ignore -autodoc_mock_imports = ["cf", "glymur", "h5netcdf", "imageio", "mipp", "netCDF4", +autodoc_mock_imports = ["cf", "glymur", "h5netcdf", "holoviews", "imageio", "mipp", "netCDF4", "pygac", "pygrib", "pyhdf", "pyninjotiff", "pyorbital", "pyspectral", "rasterio", "trollimage", "zarr"] From 612e927726e445d6459267458dec1e2e6532dc7a Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Dec 2023 10:39:45 +0200 Subject: [PATCH 123/130] Handle warnings from encoding time in CF --- satpy/tests/writer_tests/test_cf.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/satpy/tests/writer_tests/test_cf.py b/satpy/tests/writer_tests/test_cf.py index 020cb10ec3..6d1d15527b 100644 --- a/satpy/tests/writer_tests/test_cf.py +++ b/satpy/tests/writer_tests/test_cf.py @@ -247,7 +247,7 @@ def test_single_time_value(self): attrs=dict(start_time=start_time, end_time=end_time)) with TempFile() as filename: - scn.save_datasets(filename=filename, writer="cf") + scn.save_datasets(filename=filename, writer="cf", encoding={"time": {"units": "seconds since 2018-01-01"}}) with xr.open_dataset(filename, decode_cf=True) as f: np.testing.assert_array_equal(f["time"], scn["test-array"]["time"]) bounds_exp = np.array([[start_time, end_time]], dtype="datetime64[m]") @@ -264,7 +264,8 @@ def test_time_coordinate_on_a_swath(self): coords={"time": ("y", times)}, attrs=dict(start_time=times[0], end_time=times[-1])) with TempFile() as filename: - scn.save_datasets(filename=filename, writer="cf", pretty=True) + scn.save_datasets(filename=filename, writer="cf", pretty=True, + encoding={"time": {"units": "seconds since 2018-01-01"}}) with xr.open_dataset(filename, decode_cf=True) as f: np.testing.assert_array_equal(f["time"], scn["test-array"]["time"]) @@ -280,7 +281,11 @@ def test_bounds(self): attrs=dict(start_time=start_time, end_time=end_time)) with TempFile() as filename: - scn.save_datasets(filename=filename, writer="cf") + with warnings.catch_warnings(): + # The purpose is to use the default time encoding, silence the warning + warnings.filterwarnings("ignore", category=UserWarning, + message=r"Times can't be serialized faithfully to int64 with requested units") + scn.save_datasets(filename=filename, writer="cf") # Check decoded time coordinates & bounds with xr.open_dataset(filename, decode_cf=True) as f: bounds_exp = np.array([[start_time, end_time]], dtype="datetime64[m]") @@ -319,7 +324,8 @@ def test_bounds_minimum(self): attrs=dict(start_time=start_timeB, end_time=end_timeB)) with TempFile() as filename: - scn.save_datasets(filename=filename, writer="cf") + scn.save_datasets(filename=filename, writer="cf", + encoding={"time": {"units": "seconds since 2018-01-01"}}) with xr.open_dataset(filename, decode_cf=True) as f: bounds_exp = np.array([[start_timeA, end_timeB]], dtype="datetime64[m]") np.testing.assert_array_equal(f["time_bnds"], bounds_exp) @@ -340,7 +346,8 @@ def test_bounds_missing_time_info(self): dims=["x", "y", "time"], coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]}) with TempFile() as filename: - scn.save_datasets(filename=filename, writer="cf") + scn.save_datasets(filename=filename, writer="cf", + encoding={"time": {"units": "seconds since 2018-01-01"}}) with xr.open_dataset(filename, decode_cf=True) as f: bounds_exp = np.array([[start_timeA, end_timeA]], dtype="datetime64[m]") np.testing.assert_array_equal(f["time_bnds"], bounds_exp) @@ -357,7 +364,8 @@ def test_unlimited_dims_kwarg(self): attrs=dict(start_time=start_time, end_time=end_time)) with TempFile() as filename: - scn.save_datasets(filename=filename, writer="cf", unlimited_dims=["time"]) + scn.save_datasets(filename=filename, writer="cf", unlimited_dims=["time"], + encoding={"time": {"units": "seconds since 2018-01-01"}}) with xr.open_dataset(filename) as f: assert set(f.encoding["unlimited_dims"]) == {"time"} From 8f44b3112853366e9914eeafb267e243bf413fb6 Mon Sep 17 00:00:00 2001 From: Johan Strandgren Date: Fri, 15 Dec 2023 08:47:15 +0000 Subject: [PATCH 124/130] Put new invert_alpha keyword as last optional keyword to ensure backwards compatibility. --- satpy/composites/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 88375bc933..fe21623010 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -1014,7 +1014,7 @@ class CloudCompositor(GenericCompositor): """Detect clouds based on thresholding and use it as a mask for compositing.""" def __init__(self, name, transition_min=258.15, transition_max=298.15, # noqa: D417 - invert_alpha=False, transition_gamma=3.0, **kwargs): + transition_gamma=3.0, invert_alpha=False, **kwargs): """Collect custom configuration values. Args: @@ -1022,15 +1022,15 @@ def __init__(self, name, transition_min=258.15, transition_max=298.15, # noqa: clouds -> opaque white transition_max (float): Values above this are cloud free -> transparent + transition_gamma (float): Gamma correction to apply at the end invert_alpha (bool): Invert the alpha channel to make low data values transparent and high data values opaque. - transition_gamma (float): Gamma correction to apply at the end """ self.transition_min = transition_min self.transition_max = transition_max - self.invert_alpha = invert_alpha self.transition_gamma = transition_gamma + self.invert_alpha = invert_alpha super(CloudCompositor, self).__init__(name, **kwargs) def __call__(self, projectables, **kwargs): @@ -1160,8 +1160,8 @@ class LowCloudCompositor(CloudCompositor): def __init__(self, name, values_land=(1,), values_water=(0,), # noqa: D417 range_land=(0.0, 4.0), range_water=(0.0, 4.0), - invert_alpha=True, - transition_gamma=1.0, **kwargs): + transition_gamma=1.0, + invert_alpha=True, **kwargs): """Init info. Collect custom configuration values. @@ -1173,10 +1173,10 @@ def __init__(self, name, values_land=(1,), values_water=(0,), # noqa: D417 difference over land surface types. range_water (tuple): Threshold values used for masking low-level clouds from the brightness temperature difference over water. - invert_alpha (bool): Invert the alpha channel to make low data values transparent - and high data values opaque. transition_gamma (float): Gamma correction to apply to the alpha channel within the brightness temperature difference range. + invert_alpha (bool): Invert the alpha channel to make low data values transparent + and high data values opaque. """ if len(range_land) != 2: raise ValueError(f"Expected 2 `range_land` values, got {len(range_land)}") @@ -1188,7 +1188,7 @@ def __init__(self, name, values_land=(1,), values_water=(0,), # noqa: D417 self.range_land = range_land self.range_water = range_water super().__init__(name, transition_min=None, transition_max=None, - invert_alpha=invert_alpha, transition_gamma=transition_gamma, **kwargs) + transition_gamma=transition_gamma, invert_alpha=invert_alpha, **kwargs) def __call__(self, projectables, **kwargs): """Generate the composite. From 9b0bcae1169bf81297383e77806403625389d5b9 Mon Sep 17 00:00:00 2001 From: Dario Stelitano Date: Fri, 15 Dec 2023 10:44:15 +0100 Subject: [PATCH 125/130] Holoviews inside to_hvplot method --- satpy/scene.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/satpy/scene.py b/satpy/scene.py index fe6bbce1f9..d1ba795ac8 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -25,7 +25,6 @@ import numpy as np import xarray as xr -from holoviews import Overlay from pyresample.geometry import AreaDefinition, BaseDefinition, SwathDefinition from xarray import DataArray @@ -1109,6 +1108,8 @@ def _plot_quadmesh(xarray_ds, variable, **defaults): **defaults) import hvplot.xarray as hvplot_xarray # noqa + from holoviews import Overlay + plot = Overlay() xarray_ds = self.to_xarray_dataset(datasets) From 36aa47145eabb1f1bea4adb5483606edbd4946c8 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 15 Dec 2023 12:38:11 +0100 Subject: [PATCH 126/130] add a check for the presence of file_handlers attribute --- satpy/readers/yaml_reader.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index 3171449b03..5444d7e16f 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -1182,9 +1182,10 @@ def create_filehandlers(self, filenames, fh_kwargs=None): return created_fhs def _sort_segment_filehandler_by_segment_number(self): - for file_type in self.file_handlers.keys(): - self.file_handlers[file_type] = sorted(self.file_handlers[file_type], - key=lambda x: x.filename_info.get("segment", 0)) + if hasattr(self, "file_handlers"): + for file_type in self.file_handlers.keys(): + self.file_handlers[file_type] = sorted(self.file_handlers[file_type], + key=lambda x: x.filename_info.get("segment", 0)) def _load_dataset(self, dsid, ds_info, file_handlers, dim="y", pad_data=True): """Load only a piece of the dataset.""" From d326eef012c27b212a8f8734857f971156f97152 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 15 Dec 2023 15:56:43 +0100 Subject: [PATCH 127/130] match all projectables instead of only subset in NDVIHybridGreen __call__ --- satpy/composites/spectral.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/composites/spectral.py b/satpy/composites/spectral.py index d656bab7ec..138f17bd85 100644 --- a/satpy/composites/spectral.py +++ b/satpy/composites/spectral.py @@ -159,9 +159,9 @@ def __call__(self, projectables, optional_datasets=None, **attrs): LOG.info(f"Applying NDVI-weighted hybrid-green correction with limits [{self.limits[0]}, " f"{self.limits[1]}] and strength {self.strength}.") - ndvi_input = self.match_data_arrays([projectables[1], projectables[2]]) + ndvi_input = self.match_data_arrays(projectables) - ndvi = (ndvi_input[1] - ndvi_input[0]) / (ndvi_input[1] + ndvi_input[0]) + ndvi = (ndvi_input[2] - ndvi_input[1]) / (ndvi_input[2] + ndvi_input[1]) ndvi = ndvi.clip(self.ndvi_min, self.ndvi_max) From 5301dcab4141098cd78781137f784f068e96d1f9 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 15 Dec 2023 17:57:37 +0100 Subject: [PATCH 128/130] use projectables in match_data_arrays return and add test for coordinates alignment --- satpy/composites/spectral.py | 4 ++-- satpy/tests/compositor_tests/test_spectral.py | 21 ++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/satpy/composites/spectral.py b/satpy/composites/spectral.py index 138f17bd85..f7219ec94d 100644 --- a/satpy/composites/spectral.py +++ b/satpy/composites/spectral.py @@ -159,9 +159,9 @@ def __call__(self, projectables, optional_datasets=None, **attrs): LOG.info(f"Applying NDVI-weighted hybrid-green correction with limits [{self.limits[0]}, " f"{self.limits[1]}] and strength {self.strength}.") - ndvi_input = self.match_data_arrays(projectables) + projectables = self.match_data_arrays(projectables) - ndvi = (ndvi_input[2] - ndvi_input[1]) / (ndvi_input[2] + ndvi_input[1]) + ndvi = (projectables[2] - projectables[1]) / (projectables[2] + projectables[1]) ndvi = ndvi.clip(self.ndvi_min, self.ndvi_max) diff --git a/satpy/tests/compositor_tests/test_spectral.py b/satpy/tests/compositor_tests/test_spectral.py index eb3db8de5c..a68f9f2f0a 100644 --- a/satpy/tests/compositor_tests/test_spectral.py +++ b/satpy/tests/compositor_tests/test_spectral.py @@ -73,15 +73,16 @@ class TestNdviHybridGreenCompositor: def setup_method(self): """Initialize channels.""" + coord_val = [1.0, 2.0] self.c01 = xr.DataArray( da.from_array(np.array([[0.25, 0.30], [0.20, 0.30]], dtype=np.float32), chunks=25), - dims=("y", "x"), attrs={"name": "C02"}) + dims=("y", "x"), coords=[coord_val, coord_val], attrs={"name": "C02"}) self.c02 = xr.DataArray( da.from_array(np.array([[0.25, 0.30], [0.25, 0.35]], dtype=np.float32), chunks=25), - dims=("y", "x"), attrs={"name": "C03"}) + dims=("y", "x"), coords=[coord_val, coord_val], attrs={"name": "C03"}) self.c03 = xr.DataArray( da.from_array(np.array([[0.35, 0.35], [0.28, 0.65]], dtype=np.float32), chunks=25), - dims=("y", "x"), attrs={"name": "C04"}) + dims=("y", "x"), coords=[coord_val, coord_val], attrs={"name": "C04"}) def test_ndvi_hybrid_green(self): """Test General functionality with linear scaling from ndvi to blend fraction.""" @@ -123,3 +124,17 @@ def test_invalid_strength(self): with pytest.raises(ValueError, match="Expected strength greater than 0.0, got 0.0."): _ = NDVIHybridGreen("ndvi_hybrid_green", strength=0.0, prerequisites=(0.51, 0.65, 0.85), standard_name="toa_bidirectional_reflectance") + + def test_with_slightly_mismatching_coord_input(self): + """Test the case where an input (typically the red band) has a slightly different coordinate. + + If match_data_arrays is called correctly, the coords will be aligned and the array will have the expected shape. + + """ + comp = NDVIHybridGreen("ndvi_hybrid_green", limits=(0.15, 0.05), prerequisites=(0.51, 0.65, 0.85), + standard_name="toa_bidirectional_reflectance") + + c02_bad_shape = self.c02.copy() + c02_bad_shape.coords["y"] = [1.1, 2.] + res = comp((self.c01, c02_bad_shape, self.c03)) + assert res.shape == (2, 2) From fb2ec9e17d600e3ec170d11d154ed485563d527f Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 15 Dec 2023 18:00:46 +0100 Subject: [PATCH 129/130] make codefactor happy --- satpy/tests/compositor_tests/test_spectral.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/tests/compositor_tests/test_spectral.py b/satpy/tests/compositor_tests/test_spectral.py index a68f9f2f0a..2f7d9fd7cb 100644 --- a/satpy/tests/compositor_tests/test_spectral.py +++ b/satpy/tests/compositor_tests/test_spectral.py @@ -79,10 +79,10 @@ def setup_method(self): dims=("y", "x"), coords=[coord_val, coord_val], attrs={"name": "C02"}) self.c02 = xr.DataArray( da.from_array(np.array([[0.25, 0.30], [0.25, 0.35]], dtype=np.float32), chunks=25), - dims=("y", "x"), coords=[coord_val, coord_val], attrs={"name": "C03"}) + dims=("y", "x"), coords=[coord_val, coord_val], attrs={"name": "C03"}) self.c03 = xr.DataArray( da.from_array(np.array([[0.35, 0.35], [0.28, 0.65]], dtype=np.float32), chunks=25), - dims=("y", "x"), coords=[coord_val, coord_val], attrs={"name": "C04"}) + dims=("y", "x"), coords=[coord_val, coord_val], attrs={"name": "C04"}) def test_ndvi_hybrid_green(self): """Test General functionality with linear scaling from ndvi to blend fraction.""" From e20ea4182a0cf14791e40e9886c9724167f49920 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 18 Dec 2023 09:06:34 +0100 Subject: [PATCH 130/130] Update changelog for v0.46.0 --- CHANGELOG.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa85b83f56..8730209f99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,59 @@ +## Version 0.46.0 (2023/12/18) + +### Issues Closed + +* [Issue 2668](https://github.com/pytroll/satpy/issues/2668) - FCI HRFI true_color unavailable even after native resampling if upper_right_corner is used ([PR 2690](https://github.com/pytroll/satpy/pull/2690) by [@djhoese](https://github.com/djhoese)) +* [Issue 2664](https://github.com/pytroll/satpy/issues/2664) - Cannot generate day-night composites +* [Issue 2654](https://github.com/pytroll/satpy/issues/2654) - Unable to read radiance with AVHRR EPS ([PR 2655](https://github.com/pytroll/satpy/pull/2655) by [@mraspaud](https://github.com/mraspaud)) +* [Issue 2647](https://github.com/pytroll/satpy/issues/2647) - Preservation of input data dtype in processing FCI data +* [Issue 2618](https://github.com/pytroll/satpy/issues/2618) - GCOM-C Support (Continued) ([PR 1094](https://github.com/pytroll/satpy/pull/1094) by [@mraspaud](https://github.com/mraspaud)) +* [Issue 2588](https://github.com/pytroll/satpy/issues/2588) - FCI chunks/segments out of order if pad_data=False ([PR 2692](https://github.com/pytroll/satpy/pull/2692) by [@ameraner](https://github.com/ameraner)) +* [Issue 2263](https://github.com/pytroll/satpy/issues/2263) - VIIRS day composite 'snow_age' does not work with Satpy 0.37.1 +* [Issue 1496](https://github.com/pytroll/satpy/issues/1496) - Improve error reporting of satpy.utils.get_satpos +* [Issue 1086](https://github.com/pytroll/satpy/issues/1086) - Add a reader for GCOM-C Level 1 data ([PR 1094](https://github.com/pytroll/satpy/pull/1094) by [@mraspaud](https://github.com/mraspaud)) + +In this release 9 issues were closed. + +### Pull Requests Merged + +#### Bugs fixed + +* [PR 2694](https://github.com/pytroll/satpy/pull/2694) - Match all projectables in `NDVIHybridGreen.__call__` to avoid coordinate mismatch errors ([2668](https://github.com/pytroll/satpy/issues/2668), [2668](https://github.com/pytroll/satpy/issues/2668)) +* [PR 2692](https://github.com/pytroll/satpy/pull/2692) - Anticipate filehandler sorting in `GEOSegmentYAMLReader` to have sorted handlers also with `pad_data=False` ([2588](https://github.com/pytroll/satpy/issues/2588)) +* [PR 2690](https://github.com/pytroll/satpy/pull/2690) - Fix composites failing on non-aligned geolocation coordinates ([2668](https://github.com/pytroll/satpy/issues/2668)) +* [PR 2682](https://github.com/pytroll/satpy/pull/2682) - Update AHI HSD reader to correctly handle singleton arrays. +* [PR 2674](https://github.com/pytroll/satpy/pull/2674) - Update xarray version in CF writer tests for compression kwarg +* [PR 2671](https://github.com/pytroll/satpy/pull/2671) - Workaround AWIPS bug not handling integers properly in "awips_tiled" writer +* [PR 2669](https://github.com/pytroll/satpy/pull/2669) - Fix RealisticColors compositor upcasting data to float64 +* [PR 2655](https://github.com/pytroll/satpy/pull/2655) - Fix missing radiance units in eps l1b ([2654](https://github.com/pytroll/satpy/issues/2654)) + +#### Features added + +* [PR 2683](https://github.com/pytroll/satpy/pull/2683) - Fci/l2/amv/reader +* [PR 2679](https://github.com/pytroll/satpy/pull/2679) - Update MiRS reader coefficient files to newer version +* [PR 2677](https://github.com/pytroll/satpy/pull/2677) - Add remaining JPSS satellite platform aliases to "mirs" reader ([665](https://github.com/ssec/polar2grid/issues/665)) +* [PR 2669](https://github.com/pytroll/satpy/pull/2669) - Fix RealisticColors compositor upcasting data to float64 +* [PR 2660](https://github.com/pytroll/satpy/pull/2660) - Update tropomi_l2 reader with "_reduced" file patterns +* [PR 2557](https://github.com/pytroll/satpy/pull/2557) - Add baseline for GeoColor composite including FCI, AHI and ABI recipes +* [PR 2106](https://github.com/pytroll/satpy/pull/2106) - Add Scene function to use Hvplot backend visualization +* [PR 1094](https://github.com/pytroll/satpy/pull/1094) - Add Gcom-C sgli reader ([2618](https://github.com/pytroll/satpy/issues/2618), [1086](https://github.com/pytroll/satpy/issues/1086)) + +#### Backward incompatible changes + +* [PR 2684](https://github.com/pytroll/satpy/pull/2684) - Get rid of warnings in compositor tests + +#### Clean ups + +* [PR 2691](https://github.com/pytroll/satpy/pull/2691) - Reduce the number of warnings in writer tests +* [PR 2690](https://github.com/pytroll/satpy/pull/2690) - Fix composites failing on non-aligned geolocation coordinates ([2668](https://github.com/pytroll/satpy/issues/2668)) +* [PR 2684](https://github.com/pytroll/satpy/pull/2684) - Get rid of warnings in compositor tests +* [PR 2681](https://github.com/pytroll/satpy/pull/2681) - Get rid of warnings in resampler tests +* [PR 2676](https://github.com/pytroll/satpy/pull/2676) - Convert times in SEVIRI readers to nanosecond precision to silence warnings +* [PR 2658](https://github.com/pytroll/satpy/pull/2658) - Update unstable version of h5py in CI + +In this release 23 pull requests were closed. + + ## Version 0.45.0 (2023/11/29) ### Issues Closed