From f592d9d68f9a2d5927281364b896d3b690e55e9f Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 18 Mar 2024 14:24:01 +0100 Subject: [PATCH 1/7] Add support for EO-SIP AVHRR LAC data --- satpy/etc/readers/avhrr_l1b_gaclac.yaml | 1 + satpy/readers/avhrr_l1b_gaclac.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/satpy/etc/readers/avhrr_l1b_gaclac.yaml b/satpy/etc/readers/avhrr_l1b_gaclac.yaml index 484bed6797..a547815072 100644 --- a/satpy/etc/readers/avhrr_l1b_gaclac.yaml +++ b/satpy/etc/readers/avhrr_l1b_gaclac.yaml @@ -185,3 +185,4 @@ file_types: file_patterns: - '{creation_site:3s}.{transfer_mode:4s}.{platform_id:2s}.D{start_time:%y%j.S%H%M}.E{end_time:%H%M}.B{orbit_number:05d}{end_orbit_last_digits:02d}.{station:2s}' - '{subscription_prefix:10d}.{creation_site:3s}.{transfer_mode:4s}.{platform_id:2s}.D{start_time:%y%j.S%H%M}.E{end_time:%H%M}.B{orbit_number:05d}{end_orbit_last_digits:02d}.{station:2s}' + - '{platform_id:3s}_RPRO_AVH_L1B_1P_{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{orbit_number:06d}/image.l1b' diff --git a/satpy/readers/avhrr_l1b_gaclac.py b/satpy/readers/avhrr_l1b_gaclac.py index cfc3e1283e..6da23e2021 100644 --- a/satpy/readers/avhrr_l1b_gaclac.py +++ b/satpy/readers/avhrr_l1b_gaclac.py @@ -98,15 +98,17 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 if self._end_time < self._start_time: self._end_time += timedelta(days=1) self.platform_id = filename_info["platform_id"] - if self.platform_id in ["NK", "NL", "NM", "NN", "NP", "M1", "M2", - "M3"]: + if self.platform_id in ["NK", "NL", "NM", "NN", "NP", + "M1", "M2", "M3", + "N15", "N16", "N17", "N18", "N19"]: if filename_info.get("transfer_mode") == "GHRR": self.reader_class = GACKLMReader else: self.reader_class = LACKLMReader self.chn_dict = AVHRR3_CHANNEL_NAMES self.sensor = "avhrr-3" - elif self.platform_id in ["NC", "ND", "NF", "NH", "NJ"]: + elif self.platform_id in ["NC", "NE", "NF", "NG", "NH", "ND", "NJ", + "N07", "N08", "N09", "N10", "N11", "N12", "N14"]: if filename_info.get("transfer_mode") == "GHRR": self.reader_class = GACPODReader else: From 64dd1087fe703e63f38b89c7dcd7a98c9f37fd3c Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 18 Mar 2024 15:27:17 +0100 Subject: [PATCH 2/7] Guess if the format of LAC is eosip --- satpy/readers/avhrr_l1b_gaclac.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/satpy/readers/avhrr_l1b_gaclac.py b/satpy/readers/avhrr_l1b_gaclac.py index 6da23e2021..26e41387e6 100644 --- a/satpy/readers/avhrr_l1b_gaclac.py +++ b/satpy/readers/avhrr_l1b_gaclac.py @@ -98,9 +98,14 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 if self._end_time < self._start_time: self._end_time += timedelta(days=1) self.platform_id = filename_info["platform_id"] + + if len(self.platform_id) == 3: + self.reader_kwargs["eosip_header"] = True + if self.platform_id in ["NK", "NL", "NM", "NN", "NP", + "N15", "N16", "N17", "N18", "N19", "M1", "M2", "M3", - "N15", "N16", "N17", "N18", "N19"]: + "MOB", "MOA", "MOC"]: if filename_info.get("transfer_mode") == "GHRR": self.reader_class = GACKLMReader else: From f0c238ca2448588e3d4dd691e57926bf04bd90ba Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 18 Mar 2024 15:47:16 +0100 Subject: [PATCH 3/7] Add a test --- .../reader_tests/test_avhrr_l1b_gaclac.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py b/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py index dfcaff4514..42298af130 100644 --- a/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py +++ b/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py @@ -26,6 +26,7 @@ import xarray as xr GAC_PATTERN = '{creation_site:3s}.{transfer_mode:4s}.{platform_id:2s}.D{start_time:%y%j.S%H%M}.E{end_time:%H%M}.B{orbit_number:05d}{end_orbit_last_digits:02d}.{station:2s}' # noqa +EOSIP_PATTERN = '{platform_id:3s}_RPRO_AVH_L1B_1P_{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{orbit_number:06d}/image.l1b' # noqa GAC_POD_FILENAMES = ["NSS.GHRR.NA.D79184.S1150.E1337.B0008384.WI", "NSS.GHRR.NA.D79184.S2350.E0137.B0008384.WI", @@ -68,6 +69,8 @@ "NSS.FRAC.M2.D12153.S1729.E1910.B2915354.SV", "NSS.LHRR.NP.D16306.S1803.E1814.B3985555.WI"] +LAC_EOSIP_FILENAMES = ["N06_RPRO_AVH_L1B_1P_20061206T010808_20061206T012223_007961/image.l1b"] + @mock.patch("satpy.readers.avhrr_l1b_gaclac.GACLACFile.__init__", return_value=None) def _get_fh_mocked(init_mock, **attrs): @@ -138,6 +141,12 @@ def _get_fh(self, filename="NSS.GHRR.NG.D88002.S0614.E0807.B0670506.WI", filename_info = parse(GAC_PATTERN, filename) return self.GACLACFile(filename, filename_info, {}, **kwargs) + def _get_eosip_fh(self, filename, **kwargs): + """Create a file handler.""" + from trollsift import parse + filename_info = parse(EOSIP_PATTERN, filename) + return self.GACLACFile(filename, filename_info, {}, **kwargs) + def test_init(self): """Test GACLACFile initialization.""" from pygac.gac_klm import GACKLMReader @@ -161,6 +170,28 @@ def test_init(self): assert fh.start_time < fh.end_time assert fh.reader_class is reader_cls + + def test_init_eosip(self): + """Test GACLACFile initialization.""" + from pygac.lac_pod import LACPODReader + + kwargs = {"start_line": 1, + "end_line": 2, + "strip_invalid_coords": True, + "interpolate_coords": True, + "adjust_clock_drift": True, + "tle_dir": "tle_dir", + "tle_name": "tle_name", + "tle_thresh": 123, + "calibration": "calibration"} + for filenames, reader_cls in zip([LAC_EOSIP_FILENAMES], + [LACPODReader]): + for filename in filenames: + fh = self._get_eosip_fh(filename, **kwargs) + assert fh.start_time < fh.end_time + assert fh.reader_class is reader_cls + assert fh.reader_kwargs["eosip_header"] is True + def test_read_raw_data(self): """Test raw data reading.""" fh = _get_fh_mocked(reader=None, From 0bc8a429c9f036499d172ae4085a7bf631ae004b Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 18 Mar 2024 17:47:55 +0100 Subject: [PATCH 4/7] Refactor --- satpy/readers/avhrr_l1b_gaclac.py | 20 ++++++++++++------- .../reader_tests/test_avhrr_l1b_gaclac.py | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/satpy/readers/avhrr_l1b_gaclac.py b/satpy/readers/avhrr_l1b_gaclac.py index 26e41387e6..8f31fa765a 100644 --- a/satpy/readers/avhrr_l1b_gaclac.py +++ b/satpy/readers/avhrr_l1b_gaclac.py @@ -100,20 +100,16 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 self.platform_id = filename_info["platform_id"] if len(self.platform_id) == 3: - self.reader_kwargs["eosip_header"] = True + self.reader_kwargs["header_datetime"] = datetime(2000, 1, 1) - if self.platform_id in ["NK", "NL", "NM", "NN", "NP", - "N15", "N16", "N17", "N18", "N19", - "M1", "M2", "M3", - "MOB", "MOA", "MOC"]: + if self._is_avhrr3(): if filename_info.get("transfer_mode") == "GHRR": self.reader_class = GACKLMReader else: self.reader_class = LACKLMReader self.chn_dict = AVHRR3_CHANNEL_NAMES self.sensor = "avhrr-3" - elif self.platform_id in ["NC", "NE", "NF", "NG", "NH", "ND", "NJ", - "N07", "N08", "N09", "N10", "N11", "N12", "N14"]: + elif self._is_avhrr2(): if filename_info.get("transfer_mode") == "GHRR": self.reader_class = GACPODReader else: @@ -129,6 +125,16 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 self.sensor = "avhrr" self.filename_info = filename_info + def _is_avhrr2(self): + return self.platform_id in ["NC", "NE", "NF", "NG", "NH", "ND", "NJ", + "N07", "N08", "N09", "N10", "N11", "N12", "N14"] + + def _is_avhrr3(self): + return self.platform_id in ["NK", "NL", "NM", "NN", "NP", + "N15", "N16", "N17", "N18", "N19", + "M1", "M2", "M3", + "MOB", "MOA", "MOC"] + def read_raw_data(self): """Create a pygac reader and read raw data from the file.""" if self.reader is None: diff --git a/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py b/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py index 42298af130..8549cccc75 100644 --- a/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py +++ b/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py @@ -190,7 +190,7 @@ def test_init_eosip(self): fh = self._get_eosip_fh(filename, **kwargs) assert fh.start_time < fh.end_time assert fh.reader_class is reader_cls - assert fh.reader_kwargs["eosip_header"] is True + assert fh.reader_kwargs["header_datetime"] > datetime.date(1994, 11, 15) def test_read_raw_data(self): """Test raw data reading.""" From 0064e2197f5d4fabef5060fc48541831eb959345 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 18 Mar 2024 18:02:34 +0100 Subject: [PATCH 5/7] Replace header_datetime with header_date --- satpy/readers/avhrr_l1b_gaclac.py | 4 ++-- satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/satpy/readers/avhrr_l1b_gaclac.py b/satpy/readers/avhrr_l1b_gaclac.py index 8f31fa765a..96a13449f7 100644 --- a/satpy/readers/avhrr_l1b_gaclac.py +++ b/satpy/readers/avhrr_l1b_gaclac.py @@ -30,7 +30,7 @@ """ import logging -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta import dask.array as da import numpy as np @@ -100,7 +100,7 @@ def __init__(self, filename, filename_info, filetype_info, # noqa: D417 self.platform_id = filename_info["platform_id"] if len(self.platform_id) == 3: - self.reader_kwargs["header_datetime"] = datetime(2000, 1, 1) + self.reader_kwargs["header_date"] = date(2000, 1, 1) if self._is_avhrr3(): if filename_info.get("transfer_mode") == "GHRR": diff --git a/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py b/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py index 8549cccc75..e67f3cff2c 100644 --- a/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py +++ b/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py @@ -190,7 +190,7 @@ def test_init_eosip(self): fh = self._get_eosip_fh(filename, **kwargs) assert fh.start_time < fh.end_time assert fh.reader_class is reader_cls - assert fh.reader_kwargs["header_datetime"] > datetime.date(1994, 11, 15) + assert fh.reader_kwargs["header_date"] > datetime.date(1994, 11, 15) def test_read_raw_data(self): """Test raw data reading.""" From 276aa7dbeffcbb70085abac4418e377281b1806c Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 19 Mar 2024 07:28:35 +0100 Subject: [PATCH 6/7] Fix import --- satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py b/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py index e67f3cff2c..4f4e8e974a 100644 --- a/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py +++ b/satpy/tests/reader_tests/test_avhrr_l1b_gaclac.py @@ -17,7 +17,7 @@ # satpy. If not, see . """Pygac interface.""" -from datetime import datetime +from datetime import date, datetime from unittest import TestCase, mock import dask.array as da @@ -190,7 +190,7 @@ def test_init_eosip(self): fh = self._get_eosip_fh(filename, **kwargs) assert fh.start_time < fh.end_time assert fh.reader_class is reader_cls - assert fh.reader_kwargs["header_date"] > datetime.date(1994, 11, 15) + assert fh.reader_kwargs["header_date"] > date(1994, 11, 15) def test_read_raw_data(self): """Test raw data reading.""" From 5ffe15ef34327f58430c0585c5c95a0911368ef1 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 19 Mar 2024 19:28:27 +0100 Subject: [PATCH 7/7] Force newer botocore --- continuous_integration/environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index f1a89319a8..4fc7a508f2 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -47,6 +47,7 @@ dependencies: - pytest-cov - pytest-lazy-fixture - fsspec + - botocore>=1.33 - s3fs - python-geotiepoints - pooch