From 4961bbe7e41aea15d51b7a810ff2f9960b085fb2 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 6 Apr 2024 18:49:04 +0200 Subject: [PATCH 01/67] Initial commit, fixing base functionality --- src/smhi/smhi.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 7db8656d..6ac65914 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -7,7 +7,7 @@ from geopy import distance from geopy.geocoders import Nominatim from smhi.constants import TYPE_MAP -from smhi.metobs import Metobs +from smhi.metobs import Data, Parameters, Periods, Stations, Versions class SMHI: @@ -21,8 +21,7 @@ def __init__(self, type: str = "json", version: str = "1.0") -> None: version: API version """ self.type = TYPE_MAP[type] - self.client = Metobs(type) - self.client.get_parameters() + self.parameters = Parameters(Versions()) @property def parameters(self): @@ -31,7 +30,7 @@ def parameters(self): Returns: parameters """ - return self.client.parameters.data + return self.parameters.data def get_stations(self, parameter: Optional[int] = None): """Get stations from parameter. @@ -42,12 +41,12 @@ def get_stations(self, parameter: Optional[int] = None): Returns: stations """ - if self.client.parameters is None: + if self.parameters is None: logging.info("No parameters available.") return None - self.client.get_stations(parameter) - return self.client.stations.data + self.stations = Stations(self.parameters, parameter) + return self.stations.data def get_stations_from_title(self, title: Optional[str] = None): """Get stations from title. @@ -58,12 +57,12 @@ def get_stations_from_title(self, title: Optional[str] = None): Returns: stations """ - if self.client.stations is None: + if self.stations is None: logging.info("No stations available.") return None - self.client.get_stations(None, title) - return self.client.stations.data + self.stations = Stations(self.parameters, title) + return self.stations.data def find_stations_from_gps( self, parameter: int, latitude: float, longitude: float, dist: float = 0 @@ -144,9 +143,9 @@ def get_data( station: station id period: period to get """ - data, header = self.client.get_data_from_selection( - parameter=parameter, station=station, period=period - ) + self.stations = Stations(Parameters(Versions()), parameter) + self.periods = Periods(self.stations, station) + df = Data(self.periods) if interpolate > 0: # Find the station latitude and longitude information from Metobs stat = next( @@ -188,4 +187,4 @@ def get_data( > data.index.to_series().diff().median() ] data = data.sort_index() - return data, header + return df From c4fa30ad9e80baf1eab738bea830b2091667f57d Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 6 Apr 2024 21:33:05 +0200 Subject: [PATCH 02/67] Restructure get_data --- src/smhi/smhi.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 6ac65914..2bc32e74 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -145,17 +145,17 @@ def get_data( """ self.stations = Stations(Parameters(Versions()), parameter) self.periods = Periods(self.stations, station) - df = Data(self.periods) + data = Data(self.periods) if interpolate > 0: # Find the station latitude and longitude information from Metobs - stat = next( - item for item in self.client.stations.stations if item["id"] == station - ) - latitude = stat["latitude"] - longitude = stat["longitude"] - - holes_to_fill = data[ - data.index.to_series().diff() > data.index.to_series().diff().median() + # should be replaced by a self.periods.position[0].latitude + stat = next(item for item in self.stations.station if item.id == station) + latitude = stat.latitude + longitude = stat.longitude + + holes_to_fill = data.df[ + data.df.index.to_series().diff() + > data.df.index.to_series().diff().median() ] # Find stations within a given radius - set in "interpolate". self.find_stations_from_gps( @@ -167,24 +167,25 @@ def get_data( # Iterate over nearby stations, starting with the closest for nearby_station in self.nearby_stations[1:]: - tmpdata, _ = self.get_data(parameter, nearby_station[0]) + tmpdata = Data(Periods(self.stations, station)) for time, _ in holes_to_fill.iterrows(): - earliertime = data[data.index < time].index.max() + earliertime = data.df[data.df.index < time].index.max() if ( len( - tmpdata[ - (tmpdata.index > earliertime) & (tmpdata.index < time) + tmpdata.df[ + (tmpdata.df.index > earliertime) + & (tmpdata.df.index < time) ] ) > 0 ): - data = pd.concat([data, tmpdata], axis=0, join="outer") + data.df = pd.concat([data.df, tmpdata.df], axis=0, join="outer") # Re-check how many holes remain - holes_to_fill = data[ - data.index.to_series().diff() - > data.index.to_series().diff().median() + holes_to_fill = data.df[ + data.df.index.to_series().diff() + > data.df.index.to_series().diff().median() ] - data = data.sort_index() - return df + data.df = data.df.sort_index() + return data From d96bc59c0368061fbc6e4503c64e0803723a7335 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 6 Apr 2024 22:20:05 +0200 Subject: [PATCH 03/67] Fix gps station find and parallells --- src/smhi/smhi.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 2bc32e74..c34451dd 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -81,17 +81,15 @@ def find_stations_from_gps( return None user_position = (latitude, longitude) - self.get_stations(parameter) + self.stations = Stations(self.parameters, parameter) self.nearby_stations: List[Tuple[Any, Any, Any]] - all_stations = self.client.stations.stations + all_stations = self.stations.stations if dist == 0: stations = [ ( - s["id"], - s["name"], - distance.distance( - user_position, (s["latitude"], s["longitude"]) - ).km, + s.id, + s.name, + distance.distance(user_position, (s.latitude, s.longitude)).km, ) for s in all_stations ] @@ -100,15 +98,12 @@ def find_stations_from_gps( else: self.nearby_stations = [ ( - s["id"], - s["name"], - distance.distance( - user_position, (s["latitude"], s["longitude"]) - ).km, + s.id, + s.name, + distance.distance(user_position, (s.latitude, s.longitude)).km, ) for s in all_stations - if distance.distance(user_position, (s["latitude"], s["longitude"])) - <= dist + if distance.distance(user_position, (s.latitude, s.longitude)) <= dist ] self.nearby_stations = sorted(self.nearby_stations, key=lambda x: x[2]) From 16add6ae5562e4fa30311c6c45401eaf1aeb7467 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 6 Apr 2024 22:33:43 +0200 Subject: [PATCH 04/67] remove parameters() property --- src/smhi/smhi.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index c34451dd..fccecfc1 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -6,7 +6,6 @@ import pandas as pd from geopy import distance from geopy.geocoders import Nominatim -from smhi.constants import TYPE_MAP from smhi.metobs import Data, Parameters, Periods, Stations, Versions @@ -20,17 +19,8 @@ def __init__(self, type: str = "json", version: str = "1.0") -> None: type: API type version: API version """ - self.type = TYPE_MAP[type] - self.parameters = Parameters(Versions()) - - @property - def parameters(self): - """Get available parameters. - - Returns: - parameters - """ - return self.parameters.data + self.versions = Versions() + self.parameters = Parameters(self.versions) def get_stations(self, parameter: Optional[int] = None): """Get stations from parameter. From cfc459075b199d75c7e3469c9cc15d6481e11815 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sun, 7 Apr 2024 08:33:00 +0200 Subject: [PATCH 05/67] small edit --- src/smhi/smhi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index fccecfc1..ce9e1ad1 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -20,7 +20,7 @@ def __init__(self, type: str = "json", version: str = "1.0") -> None: version: API version """ self.versions = Versions() - self.parameters = Parameters(self.versions) + self.parameters = Parameters() def get_stations(self, parameter: Optional[int] = None): """Get stations from parameter. @@ -128,7 +128,7 @@ def get_data( station: station id period: period to get """ - self.stations = Stations(Parameters(Versions()), parameter) + self.stations = Stations(Parameters(), parameter) self.periods = Periods(self.stations, station) data = Data(self.periods) if interpolate > 0: From 043c823b6332ceb604a00fe892e8f37737b1b848 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sun, 7 Apr 2024 08:53:59 +0200 Subject: [PATCH 06/67] Fix find_stations_from_gps --- src/smhi/smhi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index ce9e1ad1..a4fe70a7 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -73,7 +73,7 @@ def find_stations_from_gps( user_position = (latitude, longitude) self.stations = Stations(self.parameters, parameter) self.nearby_stations: List[Tuple[Any, Any, Any]] - all_stations = self.stations.stations + all_stations = self.stations.station if dist == 0: stations = [ ( From f5e6e2c2c50b64338caf15430c8b5530e89fd6c3 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sun, 7 Apr 2024 09:30:49 +0200 Subject: [PATCH 07/67] Fix data interpolation --- src/smhi/smhi.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index a4fe70a7..7c8e1248 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -118,7 +118,6 @@ def get_data( self, parameter: int, station: int, - period: str = "corrected-archive", interpolate: int = 0, ) -> Tuple[Any, Any]: """Get data from station. @@ -126,7 +125,7 @@ def get_data( Args: parameter: data parameter station: station id - period: period to get + interpolate: station distance """ self.stations = Stations(Parameters(), parameter) self.periods = Periods(self.stations, station) @@ -134,9 +133,8 @@ def get_data( if interpolate > 0: # Find the station latitude and longitude information from Metobs # should be replaced by a self.periods.position[0].latitude - stat = next(item for item in self.stations.station if item.id == station) - latitude = stat.latitude - longitude = stat.longitude + latitude = self.periods.position[0].latitude + longitude = self.periods.position[0].longitude holes_to_fill = data.df[ data.df.index.to_series().diff() @@ -152,7 +150,7 @@ def get_data( # Iterate over nearby stations, starting with the closest for nearby_station in self.nearby_stations[1:]: - tmpdata = Data(Periods(self.stations, station)) + tmpdata = Data(Periods(self.stations, nearby_station[0])) for time, _ in holes_to_fill.iterrows(): earliertime = data.df[data.df.index < time].index.max() From fd6a992fb360500168dd1e26bde399f2712fdcf5 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sun, 7 Apr 2024 14:47:07 +0200 Subject: [PATCH 08/67] Mock unit test --- src/smhi/smhi.py | 3 +- tests/unit/_test_unit_smhi.py | 211 ---------------------------------- tests/unit/test_unit_smhi.py | 67 +++++++++++ 3 files changed, 68 insertions(+), 213 deletions(-) delete mode 100644 tests/unit/_test_unit_smhi.py create mode 100644 tests/unit/test_unit_smhi.py diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 7c8e1248..7058d8fc 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -12,11 +12,10 @@ class SMHI: """SMHI class with high-level functions.""" - def __init__(self, type: str = "json", version: str = "1.0") -> None: + def __init__(self, version: str = "1.0") -> None: """Initialise SMHI class. Args: - type: API type version: API version """ self.versions = Versions() diff --git a/tests/unit/_test_unit_smhi.py b/tests/unit/_test_unit_smhi.py deleted file mode 100644 index 84d0eded..00000000 --- a/tests/unit/_test_unit_smhi.py +++ /dev/null @@ -1,211 +0,0 @@ -"""SMHI unit tests.""" - -from unittest.mock import patch - -import pandas as pd -import pytest -from smhi.smhi import SMHI - - -class TestUnitSMHI: - """Unit tests for SMHI class.""" - - @patch("smhi.smhi.Metobs") - def test_unit_smhi_init(self, mock_requests_metobs): - """Unit test for SMHI init method. - - Args: - mock_requests_metobs: mock requests metobs object - """ - client = SMHI() - - assert client.type == "application/json" - mock_requests_metobs.assert_called_once() - mock_requests_metobs.return_value.get_parameters.assert_called_once() - - @patch("smhi.smhi.Metobs") - def test_unit_smhi_parameters(self, mock_metobs): - """Unit test for SMHI parameters method. - - Args: - mock_metobs: mock Metobs object - """ - client = SMHI() - assert client.parameters == mock_metobs.return_value.parameters.data - - @pytest.mark.parametrize( - "parameter,metobs_parameters", - [(None, None), (1, [(2, 0), (3, 0)]), (2, [(2, 0)])], - ) - @patch("smhi.smhi.Metobs") - @patch("smhi.smhi.logging.info") - def test_unit_smhi_get_stations( - self, mock_logging_info, mock_metobs, parameter, metobs_parameters - ): - """Unit test for SMHI get_stations method. - - Args: - mock_logging_info: mock logging info object - mock_metobs: mock Metobs object - parameter: parameter (int) - metobs_parameters: Metobs return parameters - """ - mock_metobs.return_value.parameters = metobs_parameters - client = SMHI() - if metobs_parameters is None: - client.get_stations(parameter) - mock_logging_info.assert_called_once() - return - - stations = client.get_stations(parameter) - assert stations == mock_metobs.return_value.stations.data - - @pytest.mark.parametrize( - "title, metobs_stations", - [ - (None, None), - ( - "Göteborg", - pd.DataFrame({"stations": "Göteborg", "data": [(2, 0), (3, 0)]}), - ), - ], - ) - @patch("smhi.smhi.Metobs") - @patch("smhi.smhi.logging.info") - def test_unit_smhi_get_stations_from_title( - self, mock_logging_info, mock_metobs, title, metobs_stations - ): - """Unit test for SMHI get_stations_from_title method. - - Args: - title: title of station - """ - mock_metobs.return_value.stations = metobs_stations - client = SMHI() - if metobs_stations is None: - client.get_stations_from_title(title) - mock_logging_info.assert_called_once() - return - - data = client.get_stations_from_title(title) - assert (data == mock_metobs.return_value.stations.data).all() - - @pytest.mark.parametrize( - "parameter,latitude,longitude,dist,metobs_stations", - [ - (None, None, None, None, None), - ( - 1, - 15, - 72, - 0, - pd.DataFrame( - { - "stations": [ - pd.DataFrame( - { - "id": [1], - "name": ["Göteborg"], - "latitude": [57.708870], - "longitude": [11.974560], - } - ) - ], - "data": [(2, 0)], - } - ), - ), - ], - ) - @patch("smhi.smhi.distance.distance") - @patch("smhi.smhi.Metobs") - @patch("smhi.smhi.logging.info") - def test_find_stations_from_gps( - self, - mock_logging_info, - mock_metobs, - mock_distance, - parameter, - latitude, - longitude, - dist, - metobs_stations, - ): - """Unit test for SMHI find_stations_from_gps method. - - Args: - mock_logging_info: mock logging info object - mock_metobs: mock Metobs object - mock_distance: mock distance object - parameter: parameter (int) - latitude: latitude (int) - longitude: longitude (int) - dist: Distance radius in which to look for stations - metobs_stations: Metobs stations - """ - mock_metobs.return_value.stations = metobs_stations - client = SMHI() - - if parameter is None: - client.find_stations_from_gps(parameter, latitude, longitude) - mock_logging_info.assert_called_once() - return - else: - client.find_stations_from_gps(parameter, latitude, longitude, dist) - mock_distance.assert_called() - assert len(client.nearby_stations[0]) > 0 - - @pytest.mark.parametrize( - "parameter,city,dist,metobs_stations", - [ - (None, None, None, None), - ( - 1, - "Göteborg", - 0, - pd.DataFrame( - { - "stations": [ - pd.DataFrame( - { - "id": [1], - "name": ["Göteborg"], - "latitude": [57.708870], - "longitude": [11.974560], - } - ) - ], - "data": [(2, 0)], - } - ), - ), - ], - ) - @patch("smhi.smhi.distance.distance") - @patch("smhi.smhi.Nominatim") - @patch("smhi.smhi.Metobs") - def test_find_stations_by_city( - self, - mock_metobs, - mock_nominatim, - mock_distance, - parameter, - city, - dist, - metobs_stations, - ): - """Unit test for SMHI find_stations_by_city method. - - Args: - mock_metobs: mock Metobs object - mock_nominatim: mock Nominatim object - mock_distance: mock distance object - parameter: parameter (int) - city: city name - dist: Distance radius in which to look for stations - metobs_stations: Metobs stations - """ - mock_metobs.return_value.stations = metobs_stations - client = SMHI() - client.find_stations_by_city(parameter, city, dist) - mock_nominatim.assert_called_once() diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py new file mode 100644 index 00000000..e65376d9 --- /dev/null +++ b/tests/unit/test_unit_smhi.py @@ -0,0 +1,67 @@ +"""SMHI unit tests.""" + +from unittest.mock import patch +import pandas as pd +import pytest +from smhi.smhi import SMHI + + +class TestUnitSMHI: + """Unit tests for SMHI class.""" + + def test_unit_smhi_init(self): + """Unit test for SMHI init method. + + Args: + mock_requests_metobs: mock requests metobs object + """ + client = SMHI() + + assert client.versions.data[0].key == "latest" + + def test_unit_smhi_parameters(self): + """Unit test for SMHI parameters method. + + Args: + mock_metobs: mock Metobs object + """ + client = SMHI() + assert client.parameters.data[0].key == "1" + + @pytest.mark.parametrize( + "parameter", + [(None), (1)], + ) + def test_unit_smhi_get_stations(self, parameter): + """Unit test for SMHI get_stations method. + + Args: + parameter: parameter (int) + """ + assert 2 == 2 + + def test_unit_smhi_get_stations_from_title(self): + """Unit test for SMHI get_stations_from_title method. + + Args: + title: title of station + """ + assert 1 == 1 + + def test_find_stations_from_gps( + self, + ): + """Unit test for SMHI find_stations_from_gps method. + + Args: + """ + assert True + + def test_find_stations_by_city( + self, + ): + """Unit test for SMHI find_stations_by_city method. + + Args: + """ + assert True From 57c3e05e32dd568093e54c936099b4cd08aa3ccc Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Sun, 7 Apr 2024 17:42:17 +0200 Subject: [PATCH 09/67] Less state mutating, one interface get_data --- src/smhi/smhi.py | 105 +++++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 7058d8fc..874c16f3 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -6,22 +6,20 @@ import pandas as pd from geopy import distance from geopy.geocoders import Nominatim -from smhi.metobs import Data, Parameters, Periods, Stations, Versions +from smhi.metobs import Data, Parameters, Periods, Stations +from smhi.models.metobs_model import MetobsLinksModel + +logger = logging.getLogger(__name__) class SMHI: """SMHI class with high-level functions.""" - def __init__(self, version: str = "1.0") -> None: - """Initialise SMHI class. - - Args: - version: API version - """ - self.versions = Versions() + def __init__(self) -> None: + """Initialise SMHI class.""" self.parameters = Parameters() - def get_stations(self, parameter: Optional[int] = None): + def get_stations(self, parameter: Optional[int] = None) -> List[MetobsLinksModel]: """Get stations from parameter. Args: @@ -31,13 +29,14 @@ def get_stations(self, parameter: Optional[int] = None): stations """ if self.parameters is None: - logging.info("No parameters available.") + logger.info("No parameters available.") return None - self.stations = Stations(self.parameters, parameter) - return self.stations.data + return Stations(self.parameters, parameter).data - def get_stations_from_title(self, title: Optional[str] = None): + def get_stations_from_title( + self, title: Optional[str] = None + ) -> List[MetobsLinksModel]: """Get stations from title. Args: @@ -50,12 +49,15 @@ def get_stations_from_title(self, title: Optional[str] = None): logging.info("No stations available.") return None - self.stations = Stations(self.parameters, title) - return self.stations.data + return Stations(self.parameters, title).data - def find_stations_from_gps( - self, parameter: int, latitude: float, longitude: float, dist: float = 0 - ) -> None: + def _find_stations_from_gps( + self, + station_response: Stations, + latitude: float, + longitude: float, + dist: float = 0, + ) -> List[Tuple[Any, Any, Any]]: """Find stations for parameter from gps location. Args: @@ -64,15 +66,11 @@ def find_stations_from_gps( longitude: longitude dist: distance from gps location. If zero (default), chooses closest. + Returns: + nearby stations """ - if parameter is None: - logging.info("Parameter needed.") - return None - user_position = (latitude, longitude) - self.stations = Stations(self.parameters, parameter) - self.nearby_stations: List[Tuple[Any, Any, Any]] - all_stations = self.stations.station + all_stations = station_response.station if dist == 0: stations = [ ( @@ -82,10 +80,10 @@ def find_stations_from_gps( ) for s in all_stations ] - self.nearby_stations = min(stations, key=lambda x: x[2]) + nearby_stations = min(stations, key=lambda x: x[2]) else: - self.nearby_stations = [ + nearby_stations = [ ( s.id, s.name, @@ -94,62 +92,71 @@ def find_stations_from_gps( for s in all_stations if distance.distance(user_position, (s.latitude, s.longitude)) <= dist ] - self.nearby_stations = sorted(self.nearby_stations, key=lambda x: x[2]) + nearby_stations = sorted(nearby_stations, key=lambda x: x[2]) - def find_stations_by_city(self, parameter: int, city: str, dist: float = 0) -> None: + return nearby_stations + + def _find_stations_by_city( + self, station_response: Stations, city: str, dist: float = 0 + ) -> List[Tuple[Any, Any, Any]]: """Find stations for parameter from city name. Args: parameter: station parameter dist: distance from city city: name of city + + Returns: + nearby stations """ geolocator = Nominatim(user_agent="ifk-smhi") loc = geolocator.geocode(city) - self.find_stations_from_gps( - parameter=parameter, - dist=dist, + return self._find_stations_from_gps( + station_response, latitude=loc.latitude, longitude=loc.longitude, + dist=dist, ) def get_data( self, parameter: int, station: int, - interpolate: int = 0, + distance: int = 0, ) -> Tuple[Any, Any]: """Get data from station. Args: parameter: data parameter station: station id - interpolate: station distance + distance: station distance + + Returns: + data """ - self.stations = Stations(Parameters(), parameter) - self.periods = Periods(self.stations, station) - data = Data(self.periods) - if interpolate > 0: + stations = Stations(Parameters(), parameter) + periods = Periods(stations, station) + data = Data(periods) + + if distance > 0: # Find the station latitude and longitude information from Metobs # should be replaced by a self.periods.position[0].latitude - latitude = self.periods.position[0].latitude - longitude = self.periods.position[0].longitude + latitude = periods.position[0].latitude + longitude = periods.position[0].longitude holes_to_fill = data.df[ data.df.index.to_series().diff() > data.df.index.to_series().diff().median() ] - # Find stations within a given radius - set in "interpolate". - self.find_stations_from_gps( - parameter=parameter, - latitude=latitude, - longitude=longitude, - dist=interpolate, + + # Find stations within a given radius - set in "distance". + nearby_stations = self._find_stations_from_gps( + stations, latitude, longitude, distance ) # Iterate over nearby stations, starting with the closest - for nearby_station in self.nearby_stations[1:]: - tmpdata = Data(Periods(self.stations, nearby_station[0])) + for nearby_station in nearby_stations[1:]: + tmpdata = Data(Periods(stations, nearby_station[0])) for time, _ in holes_to_fill.iterrows(): earliertime = data.df[data.df.index < time].index.max() @@ -169,5 +176,7 @@ def get_data( data.df.index.to_series().diff() > data.df.index.to_series().diff().median() ] + data.df = data.df.sort_index() + return data From c4dfcca355dd3edfc38ce8c3d0e3287b57d49a55 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Sun, 7 Apr 2024 17:54:56 +0200 Subject: [PATCH 10/67] City get --- src/smhi/smhi.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 874c16f3..66d5f076 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -138,6 +138,34 @@ def get_data( periods = Periods(stations, station) data = Data(periods) + return self._interpolate(self, distance, stations, periods, data) + + def get_data_by_city( + self, + parameter: int, + city: str, + distance: int = 0, + ) -> Tuple[Any, Any]: + """Get data from station. + + Args: + parameter: data parameter + city: user provided city + distance: station distance + + Returns: + data + """ + stations = Stations(Parameters(), parameter) + station = self._find_stations_by_city( + Stations(Parameters(), parameter), city, distance + )[0] + periods = Periods(stations, station[0]) + data = Data(periods) + + return self._interpolate(self, distance, stations, periods, data) + + def _interpolate(self, distance, stations, periods, data): if distance > 0: # Find the station latitude and longitude information from Metobs # should be replaced by a self.periods.position[0].latitude From 0e5c8893126517ec6eeeeb04d926c5f57506f2c8 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Sun, 7 Apr 2024 18:03:30 +0200 Subject: [PATCH 11/67] refactor interpolate --- src/smhi/smhi.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 66d5f076..f531f054 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -169,8 +169,10 @@ def _interpolate(self, distance, stations, periods, data): if distance > 0: # Find the station latitude and longitude information from Metobs # should be replaced by a self.periods.position[0].latitude - latitude = periods.position[0].latitude - longitude = periods.position[0].longitude + latitude, longitude = ( + periods.position[0].latitude, + periods.position[0].longitude, + ) holes_to_fill = data.df[ data.df.index.to_series().diff() @@ -184,20 +186,18 @@ def _interpolate(self, distance, stations, periods, data): # Iterate over nearby stations, starting with the closest for nearby_station in nearby_stations[1:]: - tmpdata = Data(Periods(stations, nearby_station[0])) + nearby_data = Data(Periods(stations, nearby_station[0])) + for time, _ in holes_to_fill.iterrows(): earliertime = data.df[data.df.index < time].index.max() + condition = (nearby_data.df.index > earliertime) & ( + nearby_data.df.index < time + ) - if ( - len( - tmpdata.df[ - (tmpdata.df.index > earliertime) - & (tmpdata.df.index < time) - ] + if len(nearby_data.df[condition]) > 0: + data.df = pd.concat( + [data.df, nearby_data.df], axis=0, join="outer" ) - > 0 - ): - data.df = pd.concat([data.df, tmpdata.df], axis=0, join="outer") # Re-check how many holes remain holes_to_fill = data.df[ From cd8fe2679b81909798efa5f847ad5ecd4ac78e00 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Sun, 7 Apr 2024 20:27:49 +0200 Subject: [PATCH 12/67] Refactor --- src/smhi/smhi.py | 73 +++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 44 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index f531f054..097ebf73 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -16,9 +16,16 @@ class SMHI: """SMHI class with high-level functions.""" def __init__(self) -> None: - """Initialise SMHI class.""" + """Initialise SMHI class. + + Raises: + ValueError + """ self.parameters = Parameters() + if self.parameters is None: + raise ValueError("No parameters available.") + def get_stations(self, parameter: Optional[int] = None) -> List[MetobsLinksModel]: """Get stations from parameter. @@ -28,10 +35,6 @@ def get_stations(self, parameter: Optional[int] = None) -> List[MetobsLinksModel Returns: stations """ - if self.parameters is None: - logger.info("No parameters available.") - return None - return Stations(self.parameters, parameter).data def get_stations_from_title( @@ -45,11 +48,7 @@ def get_stations_from_title( Returns: stations """ - if self.stations is None: - logging.info("No stations available.") - return None - - return Stations(self.parameters, title).data + return Stations(self.parameters, parameter_title=title).data def _find_stations_from_gps( self, @@ -57,53 +56,40 @@ def _find_stations_from_gps( latitude: float, longitude: float, dist: float = 0, - ) -> List[Tuple[Any, Any, Any]]: + ) -> List[Tuple[int, str, float]]: """Find stations for parameter from gps location. Args: parameter: station parameter latitude: latitude longitude: longitude - dist: distance from gps location. If zero (default), chooses closest. + dist: distance from gps location. If zero (default), chooses closest Returns: nearby stations """ - user_position = (latitude, longitude) + user_pos = (latitude, longitude) all_stations = station_response.station + nearby_stations = [ + (s.id, s.name, distance.distance(user_pos, (s.latitude, s.longitude)).km) + for s in all_stations + ] + if dist == 0: - stations = [ - ( - s.id, - s.name, - distance.distance(user_position, (s.latitude, s.longitude)).km, - ) - for s in all_stations - ] - nearby_stations = min(stations, key=lambda x: x[2]) - - else: - nearby_stations = [ - ( - s.id, - s.name, - distance.distance(user_position, (s.latitude, s.longitude)).km, - ) - for s in all_stations - if distance.distance(user_position, (s.latitude, s.longitude)) <= dist - ] - nearby_stations = sorted(nearby_stations, key=lambda x: x[2]) + return min(nearby_stations, key=lambda x: x[2]) + + nearby_stations = [x for x in nearby_stations if x[2] <= dist] - return nearby_stations + return sorted(nearby_stations, key=lambda x: x[2]) def _find_stations_by_city( self, station_response: Stations, city: str, dist: float = 0 - ) -> List[Tuple[Any, Any, Any]]: + ) -> List[Tuple[int, str, float]]: """Find stations for parameter from city name. Args: parameter: station parameter - dist: distance from city + dist: distance from city in km city: name of city Returns: @@ -111,6 +97,7 @@ def _find_stations_by_city( """ geolocator = Nominatim(user_agent="ifk-smhi") loc = geolocator.geocode(city) + return self._find_stations_from_gps( station_response, latitude=loc.latitude, @@ -129,7 +116,7 @@ def get_data( Args: parameter: data parameter station: station id - distance: station distance + distance: station distance in km Returns: data @@ -138,7 +125,7 @@ def get_data( periods = Periods(stations, station) data = Data(periods) - return self._interpolate(self, distance, stations, periods, data) + return self._interpolate(distance, stations, periods, data) def get_data_by_city( self, @@ -151,19 +138,17 @@ def get_data_by_city( Args: parameter: data parameter city: user provided city - distance: station distance + distance: station distance in km Returns: data """ stations = Stations(Parameters(), parameter) - station = self._find_stations_by_city( - Stations(Parameters(), parameter), city, distance - )[0] + station = self._find_stations_by_city(stations, city, distance)[0] periods = Periods(stations, station[0]) data = Data(periods) - return self._interpolate(self, distance, stations, periods, data) + return self._interpolate(distance, stations, periods, data) def _interpolate(self, distance, stations, periods, data): if distance > 0: From 92a43fa38b101fb8caa2a2df06a1f97a5a9abccc Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:22:18 +0200 Subject: [PATCH 13/67] Refactor interpolate --- src/smhi/smhi.py | 59 ++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 097ebf73..ce689927 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -150,46 +150,41 @@ def get_data_by_city( return self._interpolate(distance, stations, periods, data) - def _interpolate(self, distance, stations, periods, data): + def _interpolate( + self, distance: float, stations: Stations, periods: Periods, data: Data + ) -> Data: + """Interpolate data from several stations based on allowed distance.""" if distance > 0: - # Find the station latitude and longitude information from Metobs - # should be replaced by a self.periods.position[0].latitude - latitude, longitude = ( - periods.position[0].latitude, - periods.position[0].longitude, - ) + lat, lon = (periods.position[0].latitude, periods.position[0].longitude) + df = data.df + df_index = df.index.to_series() - holes_to_fill = data.df[ - data.df.index.to_series().diff() - > data.df.index.to_series().diff().median() - ] + condition = df_index.diff() > df_index.diff().median() + holes_to_fill = df[condition] - # Find stations within a given radius - set in "distance". - nearby_stations = self._find_stations_from_gps( - stations, latitude, longitude, distance + all_nearby_stations = self._find_stations_from_gps( + stations, lat, lon, distance ) - # Iterate over nearby stations, starting with the closest - for nearby_station in nearby_stations[1:]: + for nearby_station in all_nearby_stations[1:]: nearby_data = Data(Periods(stations, nearby_station[0])) + df = self._iterate_time(df, nearby_data.df, holes_to_fill) + + holes_to_fill = df[condition] - for time, _ in holes_to_fill.iterrows(): - earliertime = data.df[data.df.index < time].index.max() - condition = (nearby_data.df.index > earliertime) & ( - nearby_data.df.index < time - ) + data.df = df.sort_index() - if len(nearby_data.df[condition]) > 0: - data.df = pd.concat( - [data.df, nearby_data.df], axis=0, join="outer" - ) + return data - # Re-check how many holes remain - holes_to_fill = data.df[ - data.df.index.to_series().diff() - > data.df.index.to_series().diff().median() - ] + def _iterate_time( + self, df: pd.DataFrame, nearby_df: pd.DataFrame, holes_to_fill: pd.DataFrame + ) -> pd.DataFrame: + """Iterate over time.""" + for time, _ in holes_to_fill.iterrows(): + earliertime = df[df.index < time].index.max() + condition = (nearby_df.index > earliertime) & (nearby_df.index < time) - data.df = data.df.sort_index() + if len(nearby_df[condition]) > 0: + df = pd.concat([df, nearby_df], axis=0, join="outer") - return data + return df From 7d11797c73d04fd703c87d5495b3f725b26e56c3 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:50:32 +0200 Subject: [PATCH 14/67] Fix refactoring bug --- src/smhi/smhi.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index ce689927..bb8d91ce 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -156,11 +156,7 @@ def _interpolate( """Interpolate data from several stations based on allowed distance.""" if distance > 0: lat, lon = (periods.position[0].latitude, periods.position[0].longitude) - df = data.df - df_index = df.index.to_series() - - condition = df_index.diff() > df_index.diff().median() - holes_to_fill = df[condition] + missing_df = self._find_missing_data(data.df) all_nearby_stations = self._find_stations_from_gps( stations, lat, lon, distance @@ -168,19 +164,18 @@ def _interpolate( for nearby_station in all_nearby_stations[1:]: nearby_data = Data(Periods(stations, nearby_station[0])) - df = self._iterate_time(df, nearby_data.df, holes_to_fill) - - holes_to_fill = df[condition] + data.df = self._iterate_time(data.df, nearby_data.df, missing_df) + missing_df = self._find_missing_data(data.df) - data.df = df.sort_index() + data.df = data.df.sort_index() return data def _iterate_time( - self, df: pd.DataFrame, nearby_df: pd.DataFrame, holes_to_fill: pd.DataFrame + self, df: pd.DataFrame, nearby_df: pd.DataFrame, missing_df: pd.DataFrame ) -> pd.DataFrame: """Iterate over time.""" - for time, _ in holes_to_fill.iterrows(): + for time, _ in missing_df.iterrows(): earliertime = df[df.index < time].index.max() condition = (nearby_df.index > earliertime) & (nearby_df.index < time) @@ -188,3 +183,7 @@ def _iterate_time( df = pd.concat([df, nearby_df], axis=0, join="outer") return df + + def _find_missing_data(self, df: pd.DataFrame) -> pd.DataFrame: + """Find missing data.""" + return df[df.index.to_series().diff() > df.index.to_series().diff().median()] From 3c4737a57548a6bcc00d88052f3e7f0f9e3ba2f4 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:37:44 +0200 Subject: [PATCH 15/67] Simplify interpolate --- src/smhi/smhi.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index bb8d91ce..6aa770a9 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -154,24 +154,23 @@ def _interpolate( self, distance: float, stations: Stations, periods: Periods, data: Data ) -> Data: """Interpolate data from several stations based on allowed distance.""" - if distance > 0: - lat, lon = (periods.position[0].latitude, periods.position[0].longitude) - missing_df = self._find_missing_data(data.df) + if distance <= 0: + return data - all_nearby_stations = self._find_stations_from_gps( - stations, lat, lon, distance - ) + lat, lon = (periods.position[0].latitude, periods.position[0].longitude) + missing_df = self._find_missing_data(data.df) + all_nearby_stations = self._find_stations_from_gps(stations, lat, lon, distance) - for nearby_station in all_nearby_stations[1:]: - nearby_data = Data(Periods(stations, nearby_station[0])) - data.df = self._iterate_time(data.df, nearby_data.df, missing_df) - missing_df = self._find_missing_data(data.df) + for nearby_station in all_nearby_stations[1:]: + nearby_data = Data(Periods(stations, nearby_station[0])) + data.df = self._iterate_over_time(data.df, nearby_data.df, missing_df) + missing_df = self._find_missing_data(data.df) data.df = data.df.sort_index() return data - def _iterate_time( + def _iterate_over_time( self, df: pd.DataFrame, nearby_df: pd.DataFrame, missing_df: pd.DataFrame ) -> pd.DataFrame: """Iterate over time.""" From 7b7fceb33ad9fe25dc37e7827a48185710fbb4fc Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Thu, 11 Apr 2024 06:44:53 +0200 Subject: [PATCH 16/67] attempt at mocking __new__ --- tests/unit/test_unit_smhi.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index e65376d9..4a060a24 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -9,7 +9,9 @@ class TestUnitSMHI: """Unit tests for SMHI class.""" - def test_unit_smhi_init(self): + @patch("smhi.metobs.Versions.__new__") + @patch("smhi.metobs.Parameters.__new__") + def test_unit_smhi_init(self, mock_parameters, mock_versions): """Unit test for SMHI init method. Args: @@ -17,7 +19,8 @@ def test_unit_smhi_init(self): """ client = SMHI() - assert client.versions.data[0].key == "latest" + assert client.versions == mock_versions + assert client.parameters == mock_parameters def test_unit_smhi_parameters(self): """Unit test for SMHI parameters method. From e611d6dd8623c5dcf77fed3039fa848fd38077b3 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Wed, 10 Apr 2024 19:07:52 +0200 Subject: [PATCH 17/67] Move code --- src/smhi/smhi.py | 95 +++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 49 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 6aa770a9..f0d13f07 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -50,6 +50,51 @@ def get_stations_from_title( """ return Stations(self.parameters, parameter_title=title).data + def get_data( + self, + parameter: int, + station: int, + distance: int = 0, + ) -> Tuple[Any, Any]: + """Get data from station. + + Args: + parameter: data parameter + station: station id + distance: station distance in km + + Returns: + data + """ + stations = Stations(Parameters(), parameter) + periods = Periods(stations, station) + data = Data(periods) + + return self._interpolate(distance, stations, periods, data) + + def get_data_by_city( + self, + parameter: int, + city: str, + distance: int = 0, + ) -> Tuple[Any, Any]: + """Get data from station. + + Args: + parameter: data parameter + city: user provided city + distance: station distance in km + + Returns: + data + """ + stations = Stations(Parameters(), parameter) + station = self._find_stations_by_city(stations, city, distance)[0][0] + periods = Periods(stations, station) + data = Data(periods) + + return self._interpolate(distance, stations, periods, data) + def _find_stations_from_gps( self, station_response: Stations, @@ -99,57 +144,9 @@ def _find_stations_by_city( loc = geolocator.geocode(city) return self._find_stations_from_gps( - station_response, - latitude=loc.latitude, - longitude=loc.longitude, - dist=dist, + station_response, latitude=loc.latitude, longitude=loc.longitude, dist=dist ) - def get_data( - self, - parameter: int, - station: int, - distance: int = 0, - ) -> Tuple[Any, Any]: - """Get data from station. - - Args: - parameter: data parameter - station: station id - distance: station distance in km - - Returns: - data - """ - stations = Stations(Parameters(), parameter) - periods = Periods(stations, station) - data = Data(periods) - - return self._interpolate(distance, stations, periods, data) - - def get_data_by_city( - self, - parameter: int, - city: str, - distance: int = 0, - ) -> Tuple[Any, Any]: - """Get data from station. - - Args: - parameter: data parameter - city: user provided city - distance: station distance in km - - Returns: - data - """ - stations = Stations(Parameters(), parameter) - station = self._find_stations_by_city(stations, city, distance)[0] - periods = Periods(stations, station[0]) - data = Data(periods) - - return self._interpolate(distance, stations, periods, data) - def _interpolate( self, distance: float, stations: Stations, periods: Periods, data: Data ) -> Data: From d991de80841086b77d642c659b9869d7127e2ef0 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Tue, 16 Apr 2024 23:37:34 +0200 Subject: [PATCH 18/67] Change MetobsLinksModel to MetobsLinks --- src/smhi/smhi.py | 8 +++----- tests/unit/test_unit_smhi.py | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index f0d13f07..b8350ccb 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -7,7 +7,7 @@ from geopy import distance from geopy.geocoders import Nominatim from smhi.metobs import Data, Parameters, Periods, Stations -from smhi.models.metobs_model import MetobsLinksModel +from smhi.models.metobs_model import MetobsLinks logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ def __init__(self) -> None: if self.parameters is None: raise ValueError("No parameters available.") - def get_stations(self, parameter: Optional[int] = None) -> List[MetobsLinksModel]: + def get_stations(self, parameter: Optional[int] = None) -> List[MetobsLinks]: """Get stations from parameter. Args: @@ -37,9 +37,7 @@ def get_stations(self, parameter: Optional[int] = None) -> List[MetobsLinksModel """ return Stations(self.parameters, parameter).data - def get_stations_from_title( - self, title: Optional[str] = None - ) -> List[MetobsLinksModel]: + def get_stations_from_title(self, title: Optional[str] = None) -> List[MetobsLinks]: """Get stations from title. Args: diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 4a060a24..74b5517f 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -1,8 +1,7 @@ """SMHI unit tests.""" -from unittest.mock import patch -import pandas as pd import pytest +from unittest.mock import patch from smhi.smhi import SMHI From a6e757c449ee6d76816fcb5b00234bff3a9fcb23 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Wed, 17 Apr 2024 22:33:39 +0200 Subject: [PATCH 19/67] fix lint --- tests/unit/test_unit_smhi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 74b5517f..486a6c35 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -1,7 +1,8 @@ """SMHI unit tests.""" -import pytest from unittest.mock import patch + +import pytest from smhi.smhi import SMHI From 6ccca8b8511475e80deb669aa8c45f63b937b6bf Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Wed, 17 Apr 2024 23:06:31 +0200 Subject: [PATCH 20/67] Make _find_stations_from_gps always return list --- src/smhi/smhi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index b8350ccb..1d07fd25 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -119,7 +119,7 @@ def _find_stations_from_gps( ] if dist == 0: - return min(nearby_stations, key=lambda x: x[2]) + return [min(nearby_stations, key=lambda x: x[2])] nearby_stations = [x for x in nearby_stations if x[2] <= dist] From 2040698f8ac04b55be3805779b9a571ec6cfa881 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Thu, 18 Apr 2024 22:11:32 +0200 Subject: [PATCH 21/67] Change interpolation arg description --- src/smhi/smhi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 1d07fd25..f6313152 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -59,7 +59,7 @@ def get_data( Args: parameter: data parameter station: station id - distance: station distance in km + distance: station distance in km (for interpolation) Returns: data From 136dc525cc2874d3836460043c4bd3b111fef393 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Thu, 18 Apr 2024 22:33:25 +0200 Subject: [PATCH 22/67] Fix init test --- tests/unit/test_unit_smhi.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 486a6c35..c833659b 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -9,9 +9,8 @@ class TestUnitSMHI: """Unit tests for SMHI class.""" - @patch("smhi.metobs.Versions.__new__") @patch("smhi.metobs.Parameters.__new__") - def test_unit_smhi_init(self, mock_parameters, mock_versions): + def test_unit_smhi_init(self, mock_parameters): """Unit test for SMHI init method. Args: @@ -19,8 +18,7 @@ def test_unit_smhi_init(self, mock_parameters, mock_versions): """ client = SMHI() - assert client.versions == mock_versions - assert client.parameters == mock_parameters + assert client.parameters def test_unit_smhi_parameters(self): """Unit test for SMHI parameters method. From 0849c89d0ae0063832c96d3bdcd6edc182857431 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Thu, 18 Apr 2024 22:59:03 +0200 Subject: [PATCH 23/67] Build get_stations and get_stations_from_title tests --- tests/unit/test_unit_smhi.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index c833659b..8272ecf1 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -20,34 +20,29 @@ def test_unit_smhi_init(self, mock_parameters): assert client.parameters - def test_unit_smhi_parameters(self): - """Unit test for SMHI parameters method. - - Args: - mock_metobs: mock Metobs object - """ - client = SMHI() - assert client.parameters.data[0].key == "1" - - @pytest.mark.parametrize( - "parameter", - [(None), (1)], - ) - def test_unit_smhi_get_stations(self, parameter): + @pytest.mark.parametrize("parameter", [(None), (1)]) + @patch("smhi.metobs.Stations.__new__") + def test_unit_smhi_get_stations(self, mock_station_data, parameter): """Unit test for SMHI get_stations method. Args: parameter: parameter (int) """ - assert 2 == 2 + client = SMHI() + assert client.get_stations(parameter) - def test_unit_smhi_get_stations_from_title(self): + @pytest.mark.parametrize("parameter_title", [(None), ("Snöfall")]) + @patch("smhi.metobs.Stations.__new__") + def test_unit_smhi_get_stations_from_title( + self, mock_station_data, parameter_title + ): """Unit test for SMHI get_stations_from_title method. Args: title: title of station """ - assert 1 == 1 + client = SMHI() + assert client.get_stations_from_title(parameter_title) def test_find_stations_from_gps( self, From bb2eff6e99b9047e105e1b0819765a31ad40a372 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Thu, 18 Apr 2024 23:40:11 +0200 Subject: [PATCH 24/67] Attempted test of get_data (faulty patch) --- tests/unit/test_unit_smhi.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 8272ecf1..489a8659 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -44,6 +44,28 @@ def test_unit_smhi_get_stations_from_title( client = SMHI() assert client.get_stations_from_title(parameter_title) + @pytest.mark.parametrize( + "parameter, station, distance", [(8, 180960, None), (8, 180960, 50)] + ) + @patch("smhi.metobs.Stations.__new__") + @patch("smhi.metobs.Periods.__new__") + @patch("smhi.metobs.Data.__new__") + @patch("smhi.smhi.SMHI._interpolate") + def test_unit_get_data( + self, + mock_station_data, + mock_period_data, + mock_data_data, + mock_interpolate, + parameter, + station, + distance, + ): + mock_interpolate.return_value = "test" + client = SMHI() + data = client.get_data(parameter, station, distance) + assert data == "test" + def test_find_stations_from_gps( self, ): From 5dc2f9183b568a2e817d45f8a7b4309787dc4ac8 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Thu, 18 Apr 2024 23:52:33 +0200 Subject: [PATCH 25/67] Fixed get_data test --- tests/unit/test_unit_smhi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 489a8659..97ebfaf8 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -53,10 +53,10 @@ def test_unit_smhi_get_stations_from_title( @patch("smhi.smhi.SMHI._interpolate") def test_unit_get_data( self, - mock_station_data, - mock_period_data, - mock_data_data, mock_interpolate, + mock_data_data, + mock_period_data, + mock_station_data, parameter, station, distance, From feba25e5b0fe2f9b525d10810240dbb9b1b703e1 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Fri, 19 Apr 2024 07:15:42 +0200 Subject: [PATCH 26/67] Trying to mock Stations object --- tests/unit/test_unit_smhi.py | 69 +++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 97ebfaf8..fb936e41 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -1,9 +1,9 @@ """SMHI unit tests.""" -from unittest.mock import patch - +from unittest.mock import MagicMock, patch import pytest from smhi.smhi import SMHI +import pandas as pd class TestUnitSMHI: @@ -66,14 +66,75 @@ def test_unit_get_data( data = client.get_data(parameter, station, distance) assert data == "test" - def test_find_stations_from_gps( + @pytest.mark.parametrize( + "parameter, city, distance", [(8, "Bengtsfors", None), (8, "Bengtsfors", 50)] + ) + @patch("smhi.metobs.Stations.__new__") + @patch("smhi.metobs.Periods.__new__") + @patch("smhi.metobs.Data.__new__") + @patch("smhi.smhi.SMHI._find_stations_by_city") + @patch("smhi.smhi.SMHI._interpolate") + def test_unit_get_data_by_city( self, + mock_interpolate, + mock_find_stations_by_city, + mock_data_data, + mock_period_data, + mock_station_data, + parameter, + city, + distance, + ): + mock_interpolate.return_value = "test" + client = SMHI() + data = client.get_data_by_city(parameter, city, distance) + assert data == "test" + + @pytest.mark.parametrize( + "stations,latitude, longitude,dist", + [ + ( + MagicMock( + **{ + "id": [1, 2], + "name": ["Akalla", "Högdalen"], + "latitude": [59.5, 59.3], + "longitude": [17.8, 17.8], + } + ), + 59, + 17, + None, + ), + ( + MagicMock( + **{ + "id": [1, 2], + "name": ["Akalla", "Högdalen"], + "latitude": [59.5, 59.3], + "longitude": [17.8, 17.8], + } + ), + 59.4, + 17, + 30, + ), + ], + ) + @patch("geopy.distance") + def test_find_stations_from_gps( + self, mock_distance, stations, latitude, longitude, dist ): """Unit test for SMHI find_stations_from_gps method. Args: """ - assert True + + client = SMHI() + nearby_town = client._find_stations_from_gps( + stations, latitude, longitude, dist + ) + assert nearby_town in stations def test_find_stations_by_city( self, From 7a9cf8be391236cfbdb6fa114aad1ff805f49284 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sun, 21 Apr 2024 12:13:06 +0200 Subject: [PATCH 27/67] Still not able to properly mock Stations --- tests/unit/test_unit_smhi.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index fb936e41..2a0c7f8a 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -91,30 +91,14 @@ def test_unit_get_data_by_city( assert data == "test" @pytest.mark.parametrize( - "stations,latitude, longitude,dist", + "latitude, longitude,dist", [ ( - MagicMock( - **{ - "id": [1, 2], - "name": ["Akalla", "Högdalen"], - "latitude": [59.5, 59.3], - "longitude": [17.8, 17.8], - } - ), 59, 17, None, ), ( - MagicMock( - **{ - "id": [1, 2], - "name": ["Akalla", "Högdalen"], - "latitude": [59.5, 59.3], - "longitude": [17.8, 17.8], - } - ), 59.4, 17, 30, @@ -122,19 +106,22 @@ def test_unit_get_data_by_city( ], ) @patch("geopy.distance") - def test_find_stations_from_gps( - self, mock_distance, stations, latitude, longitude, dist - ): + def test_find_stations_from_gps(self, mock_distance, latitude, longitude, dist): """Unit test for SMHI find_stations_from_gps method. Args: """ - + stations = MagicMock() + stations.station.id.return_value = 1 + stations.station.name.return_value = "Akalla" + stations.station.latitude.return_value = 59.5 + stations.station.longitude.return_value = 17.8 + mock_distance.distance.return_value = 0 client = SMHI() nearby_town = client._find_stations_from_gps( stations, latitude, longitude, dist ) - assert nearby_town in stations + assert nearby_town[0][0] in stations.station.id def test_find_stations_by_city( self, From 3e07f72e8fb7b3e57ea911ac99692323e1d0ef34 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Tue, 23 Apr 2024 22:59:49 +0200 Subject: [PATCH 28/67] fix test_find_stations_from_gps --- tests/unit/test_unit_smhi.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 2a0c7f8a..f5e11cf7 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -90,13 +90,16 @@ def test_unit_get_data_by_city( data = client.get_data_by_city(parameter, city, distance) assert data == "test" + distanceresponse = MagicMock() + distanceresponse.km = 0 + @pytest.mark.parametrize( "latitude, longitude,dist", [ ( 59, 17, - None, + 0, ), ( 59.4, @@ -105,23 +108,25 @@ def test_unit_get_data_by_city( ), ], ) - @patch("geopy.distance") + @patch("geopy.distance.distance", return_value=distanceresponse) def test_find_stations_from_gps(self, mock_distance, latitude, longitude, dist): """Unit test for SMHI find_stations_from_gps method. Args: """ + station1 = MagicMock() + station1.id = 1 + station1.name = "Akalla" + station1.latitude = 59.5 + station1.longitude = 17.8 stations = MagicMock() - stations.station.id.return_value = 1 - stations.station.name.return_value = "Akalla" - stations.station.latitude.return_value = 59.5 - stations.station.longitude.return_value = 17.8 - mock_distance.distance.return_value = 0 + stations.station = [station1] + client = SMHI() nearby_town = client._find_stations_from_gps( stations, latitude, longitude, dist ) - assert nearby_town[0][0] in stations.station.id + assert nearby_town[0][0] == 1 def test_find_stations_by_city( self, From 354fb31f78a356d1e34c63909534742b3f38d34a Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Tue, 23 Apr 2024 23:28:34 +0200 Subject: [PATCH 29/67] test _find_stations_by_city --- tests/unit/test_unit_smhi.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index f5e11cf7..bfbfa91c 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -128,11 +128,29 @@ def test_find_stations_from_gps(self, mock_distance, latitude, longitude, dist): ) assert nearby_town[0][0] == 1 + @pytest.mark.parametrize( + "parameter, city, distance", [(8, "Bengtsfors", None), (8, "Bengtsfors", 50)] + ) + @patch("smhi.metobs.Stations.__new__") + @patch("smhi.metobs.Periods.__new__") + @patch("smhi.metobs.Data.__new__") + @patch("geopy.geocoders.Nominatim.__new__") + @patch("smhi.smhi.SMHI._find_stations_from_gps") def test_find_stations_by_city( self, + mock_find_from_gps, + mock_nominatim, + mock_data_data, + mock_period_data, + mock_station_data, + parameter, + city, + distance, ): """Unit test for SMHI find_stations_by_city method. Args: """ - assert True + client = SMHI() + data = client._find_stations_by_city(parameter, city, distance) + mock_nominatim.assert_called_once() From 38d24e15472afbbe44d547a0784d1c44587f3ebe Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Wed, 24 Apr 2024 06:21:08 +0200 Subject: [PATCH 30/67] test _interpolate --- tests/unit/test_unit_smhi.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index bfbfa91c..0cd5bce3 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -147,10 +147,40 @@ def test_find_stations_by_city( city, distance, ): - """Unit test for SMHI find_stations_by_city method. + """Unit test for SMHI _find_stations_by_city method. Args: """ client = SMHI() data = client._find_stations_by_city(parameter, city, distance) mock_nominatim.assert_called_once() + + @pytest.mark.parametrize("distance", [(0), (50)]) + @patch("smhi.metobs.Stations.__new__") + @patch("smhi.metobs.Periods.__new__") + @patch("smhi.metobs.Data.__new__") + @patch("smhi.smhi.SMHI._iterate_over_time") + @patch("smhi.smhi.SMHI._find_missing_data") + @patch("smhi.smhi.SMHI._find_stations_from_gps") + def test_interpolate( + self, + mock_find_from_gps, + mock_find_missing_data, + mock_iterate_over_time, + mock_data_data, + mock_period_data, + mock_station_data, + distance, + ): + """Unit test for SMHI _interpolate method. + + Args: + """ + client = SMHI() + data = client._interpolate( + distance, mock_station_data, mock_period_data, mock_data_data + ) + if distance <= 0: + assert data == mock_data_data + else: + mock_find_from_gps.assert_called_once() From c210bff44f5e205c25d02405a86b5880e7da137f Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Wed, 24 Apr 2024 06:59:13 +0200 Subject: [PATCH 31/67] test _iterate_over_time --- tests/unit/test_unit_smhi.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 0cd5bce3..3275e91f 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -184,3 +184,39 @@ def test_interpolate( assert data == mock_data_data else: mock_find_from_gps.assert_called_once() + + def test_iterate_over_time( + self, + ): + """Unit test for SMHI _interpolate method. + + Args: + """ + df = pd.DataFrame( + { + "date": [ + "2024-04-21 10:00", + "2024-04-21 11:00", + "2024-04-21 12:00", + "2024-04-22 12:00", + ], + "Temperatur": [1, 1, 1, 12], + } + ) + df["date"] = pd.to_datetime(df["date"]) + df = df.set_index("date") + + nearby_df = pd.DataFrame( + {"date": ["2024-04-22 10:00", "2024-04-22 11:00"], "Temperatur": [8, 9]} + ) + nearby_df["date"] = pd.to_datetime(nearby_df["date"]) + nearby_df = nearby_df.set_index("date") + + missing_df = df[ + df.index.to_series().diff() > df.index.to_series().diff().median() + ] + + client = SMHI() + + data = client._iterate_over_time(df, nearby_df, missing_df) + assert data.tail(2).iloc[0]["Temperatur"] == nearby_df.iloc[0]["Temperatur"] From 7cd1bb66fd552ae41c891220740e77984d558a5b Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Wed, 24 Apr 2024 07:02:49 +0200 Subject: [PATCH 32/67] test _find_missing_data --- tests/unit/test_unit_smhi.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 3275e91f..ad064b35 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -188,7 +188,7 @@ def test_interpolate( def test_iterate_over_time( self, ): - """Unit test for SMHI _interpolate method. + """Unit test for SMHI _iterate_over_time method. Args: """ @@ -220,3 +220,29 @@ def test_iterate_over_time( data = client._iterate_over_time(df, nearby_df, missing_df) assert data.tail(2).iloc[0]["Temperatur"] == nearby_df.iloc[0]["Temperatur"] + + def test_find_missing_data( + self, + ): + """Unit test for SMHI _find_missing_data method. + + Args: + """ + df = pd.DataFrame( + { + "date": [ + "2024-04-21 10:00", + "2024-04-21 11:00", + "2024-04-21 12:00", + "2024-04-22 12:00", + ], + "Temperatur": [1, 1, 1, 12], + } + ) + df["date"] = pd.to_datetime(df["date"]) + df = df.set_index("date") + + client = SMHI() + + missingdata = client._find_missing_data(df) + assert missingdata.iloc[0]["Temperatur"] == df.iloc[-1]["Temperatur"] From e1c2faa7984218acad932289251567b6ed35991f Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Wed, 24 Apr 2024 07:06:52 +0200 Subject: [PATCH 33/67] fix lint --- tests/unit/test_unit_smhi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index ad064b35..382cb255 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -1,10 +1,12 @@ """SMHI unit tests.""" from unittest.mock import MagicMock, patch + import pytest -from smhi.smhi import SMHI import pandas as pd +from smhi.smhi import SMHI + class TestUnitSMHI: """Unit tests for SMHI class.""" @@ -152,7 +154,7 @@ def test_find_stations_by_city( Args: """ client = SMHI() - data = client._find_stations_by_city(parameter, city, distance) + _ = client._find_stations_by_city(parameter, city, distance) mock_nominatim.assert_called_once() @pytest.mark.parametrize("distance", [(0), (50)]) From ffbf7c98978b8b7adf6434a00f53231e24d8f7be Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Wed, 24 Apr 2024 07:08:42 +0200 Subject: [PATCH 34/67] fix lint but with ruff --- tests/unit/test_unit_smhi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 382cb255..b0f6b2f6 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -2,9 +2,8 @@ from unittest.mock import MagicMock, patch -import pytest import pandas as pd - +import pytest from smhi.smhi import SMHI From ca348b0dd82623910a906c60169adea41e3f3e24 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Thu, 25 Apr 2024 19:35:55 +0200 Subject: [PATCH 35/67] Modify Metobs __init__() to stabilize integration when 'corrected-archive' isn't present --- src/smhi/metobs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index 01c00209..4fff90cb 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -306,7 +306,7 @@ class Data(BaseMetobs): def __init__( self, periods_in_station: Periods, - period: str = "corrected-archive", + period: str = "not-set", data_type: str = "json", ) -> None: """Get data from period. @@ -323,6 +323,9 @@ def __init__( """ super().__init__() + if period == "not-set": + period = periods_in_station.data[0] + if data_type != "json": raise TypeError("Only json supported.") From f1eb0cad6beae19bbdf20fa84f7d2550e33357b2 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Fri, 26 Apr 2024 23:28:37 +0200 Subject: [PATCH 36/67] add another sort to be safe --- src/smhi/metobs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index 4fff90cb..c215657a 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -324,6 +324,7 @@ def __init__( super().__init__() if period == "not-set": + periods_in_station = sorted(periods_in_station, key=lambda x: x.key) period = periods_in_station.data[0] if data_type != "json": From 2acc50c6c78f7adefff0a5c7583a6df6a5c76d3c Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 27 Apr 2024 00:34:16 +0200 Subject: [PATCH 37/67] try a typing shortcut --- src/smhi/metobs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index c215657a..d87599c1 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -324,8 +324,7 @@ def __init__( super().__init__() if period == "not-set": - periods_in_station = sorted(periods_in_station, key=lambda x: x.key) - period = periods_in_station.data[0] + period = sorted(periods_in_station, key=lambda x: x.key).data[0] if data_type != "json": raise TypeError("Only json supported.") From 58cf6d10b2b9e42dedcea55682f5000918149327 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 27 Apr 2024 00:55:35 +0200 Subject: [PATCH 38/67] fix typing again --- src/smhi/metobs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index d87599c1..53f93794 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -324,7 +324,8 @@ def __init__( super().__init__() if period == "not-set": - period = sorted(periods_in_station, key=lambda x: x.key).data[0] + available_periods = sorted(periods_in_station.data) + period = available_periods[0] if data_type != "json": raise TypeError("Only json supported.") From f1fd2651dd1a942e0cad5eb6665f9fc55f338ccf Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 27 Apr 2024 09:55:52 +0200 Subject: [PATCH 39/67] Set ordering to latest available by default --- src/smhi/metobs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index 53f93794..4ada626f 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -324,7 +324,13 @@ def __init__( super().__init__() if period == "not-set": - available_periods = sorted(periods_in_station.data) + ordering = { + "latest-hour": 0, + "latest-day": 1, + "latest-month": 2, + "corrected-archive": 3, + } + available_periods = sorted(periods_in_station.data, key=ordering.get) period = available_periods[0] if data_type != "json": From 5172cf1bcfa0580a8bae0de92be8bab981219692 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 27 Apr 2024 11:19:46 +0200 Subject: [PATCH 40/67] try fixing type issue by introducing lambda --- src/smhi/metobs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index 4ada626f..9d43106a 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -330,7 +330,9 @@ def __init__( "latest-month": 2, "corrected-archive": 3, } - available_periods = sorted(periods_in_station.data, key=ordering.get) + available_periods = sorted( + periods_in_station.data, key=lambda x: ordering.get(x) + ) period = available_periods[0] if data_type != "json": From 0fca406f927137628eb315747f8ef86a16e4a1c6 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 27 Apr 2024 11:28:43 +0200 Subject: [PATCH 41/67] Another typing bypass attempt --- src/smhi/metobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index 9d43106a..1ec3ac03 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -331,7 +331,7 @@ def __init__( "corrected-archive": 3, } available_periods = sorted( - periods_in_station.data, key=lambda x: ordering.get(x) + periods_in_station.data, key=lambda x: ordering.__getitem__(x) ) period = available_periods[0] From 924e79eb13dcc928acdd4fffd938d0e7e19c90d0 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 27 Apr 2024 14:03:46 +0200 Subject: [PATCH 42/67] fix spelling error --- src/smhi/metobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index 1ec3ac03..42a7548b 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -327,7 +327,7 @@ def __init__( ordering = { "latest-hour": 0, "latest-day": 1, - "latest-month": 2, + "latest-months": 2, "corrected-archive": 3, } available_periods = sorted( From 657915a382665e6de9265c84cd487792b15f7739 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 27 Apr 2024 21:13:33 +0200 Subject: [PATCH 43/67] Update tests/unit/test_unit_smhi.py Co-authored-by: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> --- tests/unit/test_unit_smhi.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index b0f6b2f6..b0afbc05 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -112,8 +112,6 @@ def test_unit_get_data_by_city( @patch("geopy.distance.distance", return_value=distanceresponse) def test_find_stations_from_gps(self, mock_distance, latitude, longitude, dist): """Unit test for SMHI find_stations_from_gps method. - - Args: """ station1 = MagicMock() station1.id = 1 From 226fa31095de2353332b3d50ce3ec20abfdfb448 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 27 Apr 2024 21:30:58 +0200 Subject: [PATCH 44/67] fix doc strings --- tests/unit/test_unit_smhi.py | 60 ++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index b0afbc05..4ed1bcae 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -15,7 +15,7 @@ def test_unit_smhi_init(self, mock_parameters): """Unit test for SMHI init method. Args: - mock_requests_metobs: mock requests metobs object + mock_parameters """ client = SMHI() @@ -27,6 +27,7 @@ def test_unit_smhi_get_stations(self, mock_station_data, parameter): """Unit test for SMHI get_stations method. Args: + mock_station_data, parameter: parameter (int) """ client = SMHI() @@ -40,7 +41,8 @@ def test_unit_smhi_get_stations_from_title( """Unit test for SMHI get_stations_from_title method. Args: - title: title of station + mock_station_data, + parameter_title: title of parameter """ client = SMHI() assert client.get_stations_from_title(parameter_title) @@ -62,6 +64,17 @@ def test_unit_get_data( station, distance, ): + """Unit test for SMHI method get_data + + Args: + mock_interpolate, + mock_data_data, + mock_period_data, + mock_station_data, + parameter, + station, + distance, + """ mock_interpolate.return_value = "test" client = SMHI() data = client.get_data(parameter, station, distance) @@ -86,6 +99,18 @@ def test_unit_get_data_by_city( city, distance, ): + """Unit test for SMHI _get_data_by_city method. + + args: + mock_interpolate, + mock_find_stations_by_city, + mock_data_data, + mock_period_data, + mock_station_data, + parameter, + city, + distance + """ mock_interpolate.return_value = "test" client = SMHI() data = client.get_data_by_city(parameter, city, distance) @@ -112,6 +137,12 @@ def test_unit_get_data_by_city( @patch("geopy.distance.distance", return_value=distanceresponse) def test_find_stations_from_gps(self, mock_distance, latitude, longitude, dist): """Unit test for SMHI find_stations_from_gps method. + + Args: + mock_distance, + latitude, + longitude, + dist """ station1 = MagicMock() station1.id = 1 @@ -149,6 +180,14 @@ def test_find_stations_by_city( """Unit test for SMHI _find_stations_by_city method. Args: + mock_find_from_gps, + mock_nominatim, + mock_data_data, + mock_period_data, + mock_station_data, + parameter, + city, + distance """ client = SMHI() _ = client._find_stations_by_city(parameter, city, distance) @@ -174,6 +213,13 @@ def test_interpolate( """Unit test for SMHI _interpolate method. Args: + mock_find_from_gps, + mock_find_missing_data, + mock_iterate_over_time, + mock_data_data, + mock_period_data, + mock_station_data, + distance """ client = SMHI() data = client._interpolate( @@ -187,10 +233,7 @@ def test_interpolate( def test_iterate_over_time( self, ): - """Unit test for SMHI _iterate_over_time method. - - Args: - """ + """Unit test for SMHI _iterate_over_time method.""" df = pd.DataFrame( { "date": [ @@ -223,10 +266,7 @@ def test_iterate_over_time( def test_find_missing_data( self, ): - """Unit test for SMHI _find_missing_data method. - - Args: - """ + """Unit test for SMHI _find_missing_data method.""" df = pd.DataFrame( { "date": [ From a9142f3768539df12f77c873f0180c7fa5b90c6c Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 4 May 2024 17:41:08 +0200 Subject: [PATCH 45/67] Generalize find_stations_from_gps test through mock object classes --- tests/unit/test_unit_smhi.py | 52 +++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 4ed1bcae..92e93db8 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -7,6 +7,25 @@ from smhi.smhi import SMHI +class MockMetobsStationLink: + """Support class to mock MetObs StationLink object.""" + + def __init__(self, id: int, name: str, lat: float, lon: float): + """Initiate parameters.""" + self.id = id + self.name = name + self.latitude = lat + self.longitude = lon + + +class MockParameterModel: + """Support class to mock Metobs Parameter object.""" + + def __init__(self, station: MockMetobsStationLink): + """Initiate station.""" + self.station = station + + class TestUnitSMHI: """Unit tests for SMHI class.""" @@ -120,43 +139,28 @@ def test_unit_get_data_by_city( distanceresponse.km = 0 @pytest.mark.parametrize( - "latitude, longitude,dist", - [ - ( - 59, - 17, - 0, - ), - ( - 59.4, - 17, - 30, - ), - ], + "latitude, longitude, dist, expected_result", + [(59, 17, 0, 1), (59.5, 17.75, 30, 1)], ) - @patch("geopy.distance.distance", return_value=distanceresponse) - def test_find_stations_from_gps(self, mock_distance, latitude, longitude, dist): + def test_find_stations_from_gps(self, latitude, longitude, dist, expected_result): """Unit test for SMHI find_stations_from_gps method. Args: - mock_distance, latitude, longitude, dist + expected_result """ - station1 = MagicMock() - station1.id = 1 - station1.name = "Akalla" - station1.latitude = 59.5 - station1.longitude = 17.8 - stations = MagicMock() - stations.station = [station1] + station1 = MockMetobsStationLink(1, "Akalla", 59.5, 17.8) + station2 = MockMetobsStationLink(2, "name", 57, 16) + station3 = MockMetobsStationLink(3, "name", 58, 17) + stations = MockParameterModel([station1, station2, station3]) client = SMHI() nearby_town = client._find_stations_from_gps( stations, latitude, longitude, dist ) - assert nearby_town[0][0] == 1 + assert nearby_town[0][0] == expected_result @pytest.mark.parametrize( "parameter, city, distance", [(8, "Bengtsfors", None), (8, "Bengtsfors", 50)] From a44a234693e83f2674b23217b741596ab5691a8a Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 4 May 2024 18:36:53 +0200 Subject: [PATCH 46/67] clean up test_iterate_over_time with a fixture --- tests/unit/test_unit_smhi.py | 57 +++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 92e93db8..d66e435d 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -26,6 +26,34 @@ def __init__(self, station: MockMetobsStationLink): self.station = station +@pytest.fixture +def setup_test_x(): + """Pytest fixture""" + df = pd.DataFrame( + { + "date": [ + "2024-04-21 10:00", + "2024-04-21 11:00", + "2024-04-21 12:00", + "2024-04-22 12:00", + ], + "Temperatur": [1, 1, 1, 12], + } + ) + df["date"] = pd.to_datetime(df["date"]) + df = df.set_index("date") + + nearby_df = pd.DataFrame( + {"date": ["2024-04-22 10:00", "2024-04-22 11:00"], "Temperatur": [8, 9]} + ) + nearby_df["date"] = pd.to_datetime(nearby_df["date"]) + nearby_df = nearby_df.set_index("date") + + missing_df = df[df.index.to_series().diff() > df.index.to_series().diff().median()] + + return df, nearby_df, missing_df + + class TestUnitSMHI: """Unit tests for SMHI class.""" @@ -234,34 +262,9 @@ def test_interpolate( else: mock_find_from_gps.assert_called_once() - def test_iterate_over_time( - self, - ): + def test_iterate_over_time(self, setup_test_x): """Unit test for SMHI _iterate_over_time method.""" - df = pd.DataFrame( - { - "date": [ - "2024-04-21 10:00", - "2024-04-21 11:00", - "2024-04-21 12:00", - "2024-04-22 12:00", - ], - "Temperatur": [1, 1, 1, 12], - } - ) - df["date"] = pd.to_datetime(df["date"]) - df = df.set_index("date") - - nearby_df = pd.DataFrame( - {"date": ["2024-04-22 10:00", "2024-04-22 11:00"], "Temperatur": [8, 9]} - ) - nearby_df["date"] = pd.to_datetime(nearby_df["date"]) - nearby_df = nearby_df.set_index("date") - - missing_df = df[ - df.index.to_series().diff() > df.index.to_series().diff().median() - ] - + df, nearby_df, missing_df = setup_test_x client = SMHI() data = client._iterate_over_time(df, nearby_df, missing_df) From 7c436f2677af31eeb63b9942feb1e45cb18db91f Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 4 May 2024 19:23:01 +0200 Subject: [PATCH 47/67] Clean up Metobs Data init function --- src/smhi/metobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index 42a7548b..258f3a58 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -306,7 +306,7 @@ class Data(BaseMetobs): def __init__( self, periods_in_station: Periods, - period: str = "not-set", + period: Optional[str] = None, data_type: str = "json", ) -> None: """Get data from period. @@ -323,7 +323,7 @@ def __init__( """ super().__init__() - if period == "not-set": + if period == None: ordering = { "latest-hour": 0, "latest-day": 1, From 9d8525070dc4ab76629f7c78f4dc873ac9d634f3 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 4 May 2024 19:26:39 +0200 Subject: [PATCH 48/67] Fix lint --- src/smhi/metobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index 258f3a58..f17aa0a5 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -323,7 +323,7 @@ def __init__( """ super().__init__() - if period == None: + if period is None: ordering = { "latest-hour": 0, "latest-day": 1, From 9d42f22bc66ff4160cdecb9d57979b21de5867e6 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 4 May 2024 20:03:36 +0200 Subject: [PATCH 49/67] Attempt aligning MetObs unit test with suggested structure --- tests/unit/test_unit_metobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_unit_metobs.py b/tests/unit/test_unit_metobs.py index b81acc6d..ac6189d0 100644 --- a/tests/unit/test_unit_metobs.py +++ b/tests/unit/test_unit_metobs.py @@ -509,7 +509,7 @@ def test_unit_data_init( Data(mock_periods, period, data_type) return None - if period not in METOBS_AVAILABLE_PERIODS: + if period is not None and period not in METOBS_AVAILABLE_PERIODS: with pytest.raises(NotImplementedError): Data(mock_periods, period, data_type) return None From b455417673f42d7e4bc3fbd56a701c87dd70f5fb Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 4 May 2024 20:31:34 +0200 Subject: [PATCH 50/67] add assertions to the SMHI unit testing per instructions --- tests/unit/test_unit_smhi.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index d66e435d..00b03178 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -26,6 +26,14 @@ def __init__(self, station: MockMetobsStationLink): self.station = station +class MockStationModel: + """Support class to mock Metobs Station object.""" + + def __init__(self, data="str"): + """Initiate data""" + self.data = data + + @pytest.fixture def setup_test_x(): """Pytest fixture""" @@ -57,7 +65,7 @@ def setup_test_x(): class TestUnitSMHI: """Unit tests for SMHI class.""" - @patch("smhi.metobs.Parameters.__new__") + @patch("smhi.metobs.Parameters.__new__", return_value="test") def test_unit_smhi_init(self, mock_parameters): """Unit test for SMHI init method. @@ -66,10 +74,10 @@ def test_unit_smhi_init(self, mock_parameters): """ client = SMHI() - assert client.parameters + assert client.parameters == "test" @pytest.mark.parametrize("parameter", [(None), (1)]) - @patch("smhi.metobs.Stations.__new__") + @patch("smhi.metobs.Stations.__new__", return_value=MockStationModel("Test")) def test_unit_smhi_get_stations(self, mock_station_data, parameter): """Unit test for SMHI get_stations method. @@ -78,10 +86,10 @@ def test_unit_smhi_get_stations(self, mock_station_data, parameter): parameter: parameter (int) """ client = SMHI() - assert client.get_stations(parameter) + assert client.get_stations(parameter) == "Test" @pytest.mark.parametrize("parameter_title", [(None), ("Snöfall")]) - @patch("smhi.metobs.Stations.__new__") + @patch("smhi.metobs.Stations.__new__", return_value=MockStationModel("Test")) def test_unit_smhi_get_stations_from_title( self, mock_station_data, parameter_title ): @@ -92,7 +100,7 @@ def test_unit_smhi_get_stations_from_title( parameter_title: title of parameter """ client = SMHI() - assert client.get_stations_from_title(parameter_title) + assert client.get_stations_from_title(parameter_title) == "Test" @pytest.mark.parametrize( "parameter, station, distance", [(8, 180960, None), (8, 180960, 50)] From e8651d4c59234446d251abc68228bc0c66adaed9 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Sun, 5 May 2024 07:41:10 +0200 Subject: [PATCH 51/67] Fix and simplify smhi --- src/smhi/constants.py | 12 ++++++------ src/smhi/metobs.py | 22 +++++++--------------- src/smhi/models/metobs_model.py | 3 ++- tests/unit/test_unit_smhi.py | 19 ++++++++----------- 4 files changed, 23 insertions(+), 33 deletions(-) diff --git a/src/smhi/constants.py b/src/smhi/constants.py index d36e5add..2031478d 100644 --- a/src/smhi/constants.py +++ b/src/smhi/constants.py @@ -21,12 +21,12 @@ def get_now() -> datetime: return arrow.utcnow().datetime # .isoformat("T", "seconds") + "Z" -METOBS_AVAILABLE_PERIODS = [ - "latest-hour", - "latest-day", - "latest-months", - "corrected-archive", -] +METOBS_AVAILABLE_PERIODS = { + "latest-hour": 0, + "latest-day": 1, + "latest-months": 2, + "corrected-archive": 3, +} METFCTS_URL = ( "https://opendata-download-metfcst.smhi.se/" diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index f17aa0a5..141ca62a 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -298,7 +298,7 @@ def __init__( class Data(BaseMetobs): """Get data from period for version 1 of Metobs API.""" - _metobs_available_periods: Dict[str, str] = METOBS_AVAILABLE_PERIODS + _metobs_available_periods: Dict[str, int] = METOBS_AVAILABLE_PERIODS _metobs_parameter_tim: List[str] = ["Datum", "Tid (UTC)"] _metobs_parameter_dygn: List[str] = ["Representativt dygn"] _metobs_parameter_manad: List[str] = ["Representativ månad"] @@ -323,21 +323,13 @@ def __init__( """ super().__init__() - if period is None: - ordering = { - "latest-hour": 0, - "latest-day": 1, - "latest-months": 2, - "corrected-archive": 3, - } - available_periods = sorted( - periods_in_station.data, key=lambda x: ordering.__getitem__(x) - ) - period = available_periods[0] - if data_type != "json": raise TypeError("Only json supported.") + if period is None: + period = periods_in_station.data[0] + logger.info("No period selected, selecting: {period}.") + if self._check_available_periods(periods_in_station.data, period): logger.info( "Found only one period to download. " @@ -347,8 +339,8 @@ def __init__( if period not in self._metobs_available_periods: raise NotImplementedError( - "Select a supported periods: }" - + ", ".join([p for p in self._metobs_available_periods]) + "Select a supported period: " + + ", ".join([p for p in self._metobs_available_periods.keys()]) ) self.periods_in_station = periods_in_station diff --git a/src/smhi/models/metobs_model.py b/src/smhi/models/metobs_model.py index a482ebd7..4743e592 100644 --- a/src/smhi/models/metobs_model.py +++ b/src/smhi/models/metobs_model.py @@ -4,6 +4,7 @@ import pandas as pd from pydantic import BaseModel, ConfigDict, Field, field_validator +from smhi.constants import METOBS_AVAILABLE_PERIODS class MetobsLink(BaseModel): @@ -183,7 +184,7 @@ class MetobsStationModel(MetobsBaseModel): @field_validator("period") @classmethod def serialise_period_in_order(cls, period: List[MetobsLinks]): - return sorted(period, key=lambda x: x.key) + return sorted(period, key=lambda x: METOBS_AVAILABLE_PERIODS[x.key]) @property def data(self) -> Tuple[Optional[str], ...]: diff --git a/tests/unit/test_unit_smhi.py b/tests/unit/test_unit_smhi.py index 00b03178..23951b15 100644 --- a/tests/unit/test_unit_smhi.py +++ b/tests/unit/test_unit_smhi.py @@ -35,8 +35,8 @@ def __init__(self, data="str"): @pytest.fixture -def setup_test_x(): - """Pytest fixture""" +def setup_iterate_over_time(): + """Iterate over time fixture.""" df = pd.DataFrame( { "date": [ @@ -108,7 +108,7 @@ def test_unit_smhi_get_stations_from_title( @patch("smhi.metobs.Stations.__new__") @patch("smhi.metobs.Periods.__new__") @patch("smhi.metobs.Data.__new__") - @patch("smhi.smhi.SMHI._interpolate") + @patch("smhi.smhi.SMHI._interpolate", return_value="test") def test_unit_get_data( self, mock_interpolate, @@ -130,7 +130,6 @@ def test_unit_get_data( station, distance, """ - mock_interpolate.return_value = "test" client = SMHI() data = client.get_data(parameter, station, distance) assert data == "test" @@ -142,7 +141,7 @@ def test_unit_get_data( @patch("smhi.metobs.Periods.__new__") @patch("smhi.metobs.Data.__new__") @patch("smhi.smhi.SMHI._find_stations_by_city") - @patch("smhi.smhi.SMHI._interpolate") + @patch("smhi.smhi.SMHI._interpolate", return_value="test") def test_unit_get_data_by_city( self, mock_interpolate, @@ -166,7 +165,6 @@ def test_unit_get_data_by_city( city, distance """ - mock_interpolate.return_value = "test" client = SMHI() data = client.get_data_by_city(parameter, city, distance) assert data == "test" @@ -232,6 +230,7 @@ def test_find_stations_by_city( client = SMHI() _ = client._find_stations_by_city(parameter, city, distance) mock_nominatim.assert_called_once() + mock_find_from_gps.assert_called_once() @pytest.mark.parametrize("distance", [(0), (50)]) @patch("smhi.metobs.Stations.__new__") @@ -270,17 +269,15 @@ def test_interpolate( else: mock_find_from_gps.assert_called_once() - def test_iterate_over_time(self, setup_test_x): + def test_iterate_over_time(self, setup_iterate_over_time): """Unit test for SMHI _iterate_over_time method.""" - df, nearby_df, missing_df = setup_test_x + df, nearby_df, missing_df = setup_iterate_over_time client = SMHI() data = client._iterate_over_time(df, nearby_df, missing_df) assert data.tail(2).iloc[0]["Temperatur"] == nearby_df.iloc[0]["Temperatur"] - def test_find_missing_data( - self, - ): + def test_find_missing_data(self): """Unit test for SMHI _find_missing_data method.""" df = pd.DataFrame( { From fdc1ef09b8c8efbdf204c35174264d48de2c0321 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Sun, 5 May 2024 07:44:53 +0200 Subject: [PATCH 52/67] Update metobs data test for period=None --- tests/unit/test_unit_metobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_unit_metobs.py b/tests/unit/test_unit_metobs.py index ac6189d0..2e633ffb 100644 --- a/tests/unit/test_unit_metobs.py +++ b/tests/unit/test_unit_metobs.py @@ -477,7 +477,7 @@ class TestUnitData: @pytest.mark.parametrize( "period, data_type", [ - (None, "yaml"), + ("latest", "yaml"), (None, "json"), ("latest", "json"), ("corrected-archive", "json"), @@ -509,7 +509,7 @@ def test_unit_data_init( Data(mock_periods, period, data_type) return None - if period is not None and period not in METOBS_AVAILABLE_PERIODS: + if period not in METOBS_AVAILABLE_PERIODS and period is not None: with pytest.raises(NotImplementedError): Data(mock_periods, period, data_type) return None From 4f6f323d00d811300d61f471c6560e9aadcbd5d1 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Sun, 5 May 2024 07:52:34 +0200 Subject: [PATCH 53/67] Fix period order in metobs --- src/smhi/constants.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/smhi/constants.py b/src/smhi/constants.py index 2031478d..6a438ec8 100644 --- a/src/smhi/constants.py +++ b/src/smhi/constants.py @@ -22,10 +22,10 @@ def get_now() -> datetime: METOBS_AVAILABLE_PERIODS = { - "latest-hour": 0, - "latest-day": 1, - "latest-months": 2, - "corrected-archive": 3, + "corrected-archive": 0, + "latest-months": 1, + "latest-day": 2, + "latest-hour": 3, } METFCTS_URL = ( From ad2e1da2cc0d4b9e0632b14f9778e5b608093be6 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Sun, 5 May 2024 08:19:25 +0200 Subject: [PATCH 54/67] Add delay after test avoiding API limit, if any --- tests/integration/test_integration_mesan.py | 16 ++++++++++++++++ tests/integration/test_integration_strang.py | 6 ++++++ 2 files changed, 22 insertions(+) diff --git a/tests/integration/test_integration_mesan.py b/tests/integration/test_integration_mesan.py index ebc4e5d3..6bdd3b24 100644 --- a/tests/integration/test_integration_mesan.py +++ b/tests/integration/test_integration_mesan.py @@ -1,5 +1,7 @@ """Mesan integration tests.""" +import time + import arrow import pytest from smhi.constants import MESAN_PARAMETER_DESCRIPTIONS @@ -32,6 +34,8 @@ def test_integration_mesan_approved_time(self): approved_time = client.approved_time.approved_time assert datetime_between_day(approved_time) + time.sleep(1) + def test_integration_mesan_parameters(self): """Integration test for parameters property.""" client = Mesan() @@ -41,6 +45,8 @@ def test_integration_mesan_parameters(self): ) assert len(parameters.parameter) > NUM_PARAMETERS + time.sleep(1) + def test_integration_mesan_valid_time(self): """Integration test for approved time property.""" client = Mesan() @@ -50,6 +56,8 @@ def test_integration_mesan_valid_time(self): assert len(valid_time) == NUM_VALID_TIME + time.sleep(1) + def test_integration_mesan_geo_polygon(self): """Integration test for geo_polygon property.""" client = Mesan() @@ -57,6 +65,8 @@ def test_integration_mesan_geo_polygon(self): assert geo_polygon.type_ == GEO_POLYGON_TYPE assert len(geo_polygon.coordinates[0]) > NUM_GEO_COORDINATES + time.sleep(1) + @pytest.mark.parametrize("downsample", [(1)]) def test_integration_mesan_get_geo_multipoint(self, downsample): """Integration test for get_geo_multipoint method.""" @@ -65,6 +75,8 @@ def test_integration_mesan_get_geo_multipoint(self, downsample): assert geo_multipoint.type_ == GEO_MULTIPOINT_TYPE assert len(geo_multipoint.coordinates) > NUM_GEO_COORDINATES + time.sleep(1) + @pytest.mark.parametrize("lat, lon", [(58, 16)]) def test_integration_mesan_get_point(self, lat, lon): """Integration test for get_point method.""" @@ -75,6 +87,8 @@ def test_integration_mesan_get_point(self, lat, lon): assert not point.df.empty assert point.df["t"].iloc[0] > MIN_TEMPERATURE + time.sleep(1) + @pytest.mark.parametrize( "validtime, parameter, level_type, level, geo, downsample", [ @@ -99,3 +113,5 @@ def test_integration_mesan_get_multipoint( assert not multipoint.df.empty assert multipoint.df["value"].iloc[0] > MIN_TEMPERATURE + + time.sleep(1) diff --git a/tests/integration/test_integration_strang.py b/tests/integration/test_integration_strang.py index 2ec17995..e37662d8 100644 --- a/tests/integration/test_integration_strang.py +++ b/tests/integration/test_integration_strang.py @@ -1,5 +1,7 @@ """Strang integration tests.""" +import time + import pandas as pd import pytest from smhi.strang import Strang @@ -90,6 +92,8 @@ def test_integration_strang_point( else: assert expected_result == point_model.df.index.name + time.sleep(1) + def test_integration_strang_multipoint(self, get_multipoint): """Strang MultiPoint integration tests. @@ -113,3 +117,5 @@ def test_integration_strang_multipoint(self, get_multipoint): get_multipoint, multipoint_model.df.iloc[:10, :], ) + + time.sleep(1) From 6eb611687f117cfb39380d02198ad8d2bdaf9ed8 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sun, 5 May 2024 16:08:12 +0200 Subject: [PATCH 55/67] typo in metobs, starting to build integration test --- src/smhi/metobs.py | 2 +- tests/integration/_test_integration_smhi.py | 9 ------ tests/integration/test_integration_smhi.py | 31 +++++++++++++++++++++ 3 files changed, 32 insertions(+), 10 deletions(-) delete mode 100644 tests/integration/_test_integration_smhi.py create mode 100644 tests/integration/test_integration_smhi.py diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index 4d18bf28..97df33f6 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -328,7 +328,7 @@ def __init__( if period is None: period = periods_in_station.data[0] - logger.info("No period selected, selecting: {period}.") + logger.info(f"No period selected, selecting: {period}.") if self._check_available_periods(periods_in_station.data, period): logger.info( diff --git a/tests/integration/_test_integration_smhi.py b/tests/integration/_test_integration_smhi.py deleted file mode 100644 index 02776bd4..00000000 --- a/tests/integration/_test_integration_smhi.py +++ /dev/null @@ -1,9 +0,0 @@ -"""SMHI integration tests.""" - - -class TestIntegrationSMHI: - """Integration test of SMHI class.""" - - def test_integration_smhi(self): - """Integration test of SMHI class.""" - pass diff --git a/tests/integration/test_integration_smhi.py b/tests/integration/test_integration_smhi.py new file mode 100644 index 00000000..cb01b569 --- /dev/null +++ b/tests/integration/test_integration_smhi.py @@ -0,0 +1,31 @@ +"""SMHI integration tests.""" + + +class TestIntegrationSMHI: + """Integration test of SMHI class.""" + + + @pytest.mark.parametrize( + "parameter, station, period, parameter_data, station_data, period_data, data_title, data_df", + [ + ( + 1, + 1, + "corrected-archive", + MetobsVersionItem( + key="1", + title="Lufttemperatur", + summary="momentanvärde, 1 gång/tim", + unit="celsius", + ), + (1, "Akalla"), + "corrected-archive", + "Lufttemperatur - Akalla - Kvalitetskontrollerade historiska " + + "data (utom de senaste 3 mån): Ladda ner data", + {}, + ), + ], + ) + def test_integration_smhi(self): + """Integration test of SMHI class.""" + From 52f64ad2bb46b3843134e4034d4cea8f119247dd Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Fri, 10 May 2024 05:56:57 +0200 Subject: [PATCH 56/67] Initial integration test status --- tests/integration/test_integration_smhi.py | 99 ++++++++++++++++++---- 1 file changed, 81 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_integration_smhi.py b/tests/integration/test_integration_smhi.py index cb01b569..663249db 100644 --- a/tests/integration/test_integration_smhi.py +++ b/tests/integration/test_integration_smhi.py @@ -1,31 +1,94 @@ """SMHI integration tests.""" +from unittest.mock import patch + +import pandas as pd +import pytest +from smhi.smhi import SMHI + + +class MockMetobsStationLink: + """Support class to mock MetObs StationLink object.""" + + def __init__(self, id: int, name: str, lat: float, lon: float): + """Initiate parameters.""" + self.id = id + self.name = name + self.latitude = lat + self.longitude = lon + + +class MockParameterModel: + """Support class to mock Metobs Parameter object.""" + + def __init__(self, station: MockMetobsStationLink): + """Initiate station.""" + self.station = station + + +class MockDataModel: + """Support class to mock Metobs Data object.""" + + def __init__(self, station: MockMetobsStationLink, parameter: MockParameterModel): + """initiate data.""" + df = pd.DataFrame( + { + "Lufttemperatur": [19.0, 18.2, 15.3, 17.0, 18.8], + "Kvalitet": ["G", "G", "G", "G", "G"], + }, + index=pd.to_datetime( + [ + "1859-08-01 07:00:00+00:00", + "1859-08-01 13:00:00+00:00", + "1859-08-01 20:00:00+00:00", + "1859-08-02 07:00:00+00:00", + "1859-08-02 13:00:00+00:00", + ], + ), + ) + self.df = df + class TestIntegrationSMHI: """Integration test of SMHI class.""" - - @pytest.mark.parametrize( - "parameter, station, period, parameter_data, station_data, period_data, data_title, data_df", + @pytest.mark.parametrize( + "parameter, city, period,radius,expected_result", [ ( 1, - 1, - "corrected-archive", - MetobsVersionItem( - key="1", - title="Lufttemperatur", - summary="momentanvärde, 1 gång/tim", - unit="celsius", - ), - (1, "Akalla"), - "corrected-archive", - "Lufttemperatur - Akalla - Kvalitetskontrollerade historiska " - + "data (utom de senaste 3 mån): Ladda ner data", - {}, + "Göteborg", + None, + None, + "Result", ), ], ) - def test_integration_smhi(self): + @patch( + "smhi.metobs.Stations.__new__", + return_value=MockParameterModel([MockMetobsStationLink(1, "Göteborg", 16, 57)]), + ) + @patch("smhi.metobs.Periods.__new__") + @patch( + "smhi.metobs.Data.__new__", + return_value=MockDataModel( + MockMetobsStationLink(1, "Göteborg", 16, 57), + MockParameterModel(MockMetobsStationLink(1, "Göteborg", 16, 57)), + ), + ) + def test_integration_smhi( + self, + mock_data, + mock_periods, + mock_stations, + parameter, + city, + period, + radius, + expected_result, + ): """Integration test of SMHI class.""" - + + client = SMHI() + data = client.get_data_by_city(parameter, city) + assert len(data.df["Lufttemperatur"]) == 5 From a3739af1b2f47c925b373d626fc494b1b8a30d41 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 18 May 2024 14:36:13 +0200 Subject: [PATCH 57/67] Change to full integration test mode --- src/smhi/smhi.py | 6 ++ tests/integration/test_integration_smhi.py | 76 ++++------------------ 2 files changed, 19 insertions(+), 63 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index f6313152..10d18a27 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -111,6 +111,9 @@ def _find_stations_from_gps( Returns: nearby stations """ + if not dist: + dist = 0 + user_pos = (latitude, longitude) all_stations = station_response.station nearby_stations = [ @@ -149,6 +152,9 @@ def _interpolate( self, distance: float, stations: Stations, periods: Periods, data: Data ) -> Data: """Interpolate data from several stations based on allowed distance.""" + if not distance: + return data + if distance <= 0: return data diff --git a/tests/integration/test_integration_smhi.py b/tests/integration/test_integration_smhi.py index 663249db..5fcf9534 100644 --- a/tests/integration/test_integration_smhi.py +++ b/tests/integration/test_integration_smhi.py @@ -1,52 +1,8 @@ """SMHI integration tests.""" - -from unittest.mock import patch - import pandas as pd import pytest from smhi.smhi import SMHI - - -class MockMetobsStationLink: - """Support class to mock MetObs StationLink object.""" - - def __init__(self, id: int, name: str, lat: float, lon: float): - """Initiate parameters.""" - self.id = id - self.name = name - self.latitude = lat - self.longitude = lon - - -class MockParameterModel: - """Support class to mock Metobs Parameter object.""" - - def __init__(self, station: MockMetobsStationLink): - """Initiate station.""" - self.station = station - - -class MockDataModel: - """Support class to mock Metobs Data object.""" - - def __init__(self, station: MockMetobsStationLink, parameter: MockParameterModel): - """initiate data.""" - df = pd.DataFrame( - { - "Lufttemperatur": [19.0, 18.2, 15.3, 17.0, 18.8], - "Kvalitet": ["G", "G", "G", "G", "G"], - }, - index=pd.to_datetime( - [ - "1859-08-01 07:00:00+00:00", - "1859-08-01 13:00:00+00:00", - "1859-08-01 20:00:00+00:00", - "1859-08-02 07:00:00+00:00", - "1859-08-02 13:00:00+00:00", - ], - ), - ) - self.df = df +import time class TestIntegrationSMHI: @@ -60,27 +16,19 @@ class TestIntegrationSMHI: "Göteborg", None, None, - "Result", + ["Lufttemperatur", 72630], + ), + ( + 5, + "Göteborg", + None, + 20, + ["Nederbördsmängd", 72630], ), ], ) - @patch( - "smhi.metobs.Stations.__new__", - return_value=MockParameterModel([MockMetobsStationLink(1, "Göteborg", 16, 57)]), - ) - @patch("smhi.metobs.Periods.__new__") - @patch( - "smhi.metobs.Data.__new__", - return_value=MockDataModel( - MockMetobsStationLink(1, "Göteborg", 16, 57), - MockParameterModel(MockMetobsStationLink(1, "Göteborg", 16, 57)), - ), - ) def test_integration_smhi( self, - mock_data, - mock_periods, - mock_stations, parameter, city, period, @@ -90,5 +38,7 @@ def test_integration_smhi( """Integration test of SMHI class.""" client = SMHI() - data = client.get_data_by_city(parameter, city) - assert len(data.df["Lufttemperatur"]) == 5 + data = client.get_data_by_city(parameter, city, radius) + assert len(data.df[expected_result[0]]) > 0 + assert data.station["Stationsnummer"][0] == expected_result[1] + time.sleep(1) From 6c9310e1edfc77a2f87e475dc2d54166a850c90a Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 18 May 2024 14:39:32 +0200 Subject: [PATCH 58/67] fix lint --- tests/integration/test_integration_smhi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_integration_smhi.py b/tests/integration/test_integration_smhi.py index 5fcf9534..caae0745 100644 --- a/tests/integration/test_integration_smhi.py +++ b/tests/integration/test_integration_smhi.py @@ -1,8 +1,8 @@ """SMHI integration tests.""" -import pandas as pd +import time + import pytest from smhi.smhi import SMHI -import time class TestIntegrationSMHI: From 02b604ad00daee6e83c695c03c0526a58300587b Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 18 May 2024 15:01:38 +0200 Subject: [PATCH 59/67] Reformat with ruff --- tests/integration/test_integration_smhi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_integration_smhi.py b/tests/integration/test_integration_smhi.py index caae0745..6ff39d34 100644 --- a/tests/integration/test_integration_smhi.py +++ b/tests/integration/test_integration_smhi.py @@ -1,4 +1,5 @@ """SMHI integration tests.""" + import time import pytest @@ -36,7 +37,6 @@ def test_integration_smhi( expected_result, ): """Integration test of SMHI class.""" - client = SMHI() data = client.get_data_by_city(parameter, city, radius) assert len(data.df[expected_result[0]]) > 0 From 3bd8a4b52bc1dfd2d3dec78416186925f4495f71 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 18 May 2024 16:30:35 +0200 Subject: [PATCH 60/67] Update documentation with SMHI class example --- docs/index.md | 6 ++++ docs/smhi-example.md | 80 ++++++++++++++++++++++++++++++++++++++++++ docs/smhi-reference.md | 3 ++ src/smhi/smhi.py | 2 ++ 4 files changed, 91 insertions(+) create mode 100644 docs/smhi-example.md create mode 100644 docs/smhi-reference.md diff --git a/docs/index.md b/docs/index.md index 6431cf8f..3cf94d2e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,6 +20,10 @@ Currently, only these four APIs are supported You can reach example usage of the different clients - [example of Metobs use](/ifk-smhi/metobs-example/) +- [example of Mesan use](/ifk-smhi/mesan-example/) +- [example of Metfcts use](/ifk-smhi/metfcts-example/) +- [example of Strang use](/ifk-smhi/strang-example/) +- [example of SMHI use](/ifk-smhi/smhi-example/) ## Install @@ -40,6 +44,8 @@ pip install git+https://github.com/Ingenjorsarbete-For-Klimatet/ifk-smhi.git@mai The SMHI client is in an experimental state. As of now, it only controls the Metobs sub-client. It will be expanded in future versions. +See [example of SMHI use](/ifk-smhi/smhi-example/) +for details on how to use the client. ## Metobs client diff --git a/docs/smhi-example.md b/docs/smhi-example.md new file mode 100644 index 00000000..7ea2c32e --- /dev/null +++ b/docs/smhi-example.md @@ -0,0 +1,80 @@ +# Example of SMHI direct use + +Direct usage of `SMHI`. + +## Basic use + +The `SMHI` client is in an experimental state and currently only deals with `Metobs` +data. It is intended as a parent class for easy traversing of SMHI observational +data from GPS or city names directly. + +The most basic usage of the client is to simply ask for data for a given station +and a given parameter: + +```python +from smhi.smhi import SMHI + +client = SMHI() + +client.parameters.data #List all available parameters +#Parameter 1 is hourly air temperature + +stations = client.get_stations(1) +stations.data #List all available stations for parameter 1 + +data = client.get_data(1,72630) #Get data from specific station +#Station 72630 is Gothenburg +``` + +The `data` variable holds four dataframes when a single station +is used to fetch data as above. +They are called `station`, `parameter`, `period` and `df`, where the latter holds +the actual observational data. See further +[example of Metobs use](/ifk-smhi/metobs-example/). + +## Interpolation + +When there are several stations close to a measurement location and historical data +records are incomplete, it is convenient to use nearby stations to fill in the missing +data: This naive interpolation feature is included in the toolbox by the parameter +`radius`. Thus, if we want to fetch historical data on the snow coverage in Kiruna, +the call is: + +```python +from smhi.smhi import SMHI + +client = SMHI() + +#Parameter 8 is depth of snow coverage +data = client.get_data(8,180960,40) #Get data from specific station +#Station 180960 is Kiruna. 40 indicates we will use stations within +#a 40km radius to complement any data losses. + +data2 = client.get_data(8,180960) #Get comparison data without the +#interpolation. + +import matplotlib.pyplot as plt +plt.plot(data.df.index,data.df["Snödjup"]) +plt.plot(data2.df.index,data2.df["Snödjup"]) +plt.ylabel("Snödjup") +plt.legend(["Interpolated (40km)","Station 180960"]) +plt.show() +``` + +## Finding data from a city + +There are a lot of stations available: Frequently, we are simply interested +in meterological observations from a specific city. Using the library `geopy`, +this is included in the toolbox too: + +```python +from smhi.smhi import SMHI + +client = SMHI() + +#Parameter 8 is depth of snow coverage +data = client.get_data_by_city(8,"Kiruna",40) #Get data from a station +#close to the city centre of Kiruna, and fill any empty holes with data +#from stations within a 40km perimeter + +``` diff --git a/docs/smhi-reference.md b/docs/smhi-reference.md new file mode 100644 index 00000000..c0d06a12 --- /dev/null +++ b/docs/smhi-reference.md @@ -0,0 +1,3 @@ +# SMHI reference + +::: smhi.smhi.SMHI diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index 10d18a27..ce4faef2 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -1,5 +1,6 @@ """Read SMHI data.""" +import time import logging from typing import Any, List, Optional, Tuple @@ -163,6 +164,7 @@ def _interpolate( all_nearby_stations = self._find_stations_from_gps(stations, lat, lon, distance) for nearby_station in all_nearby_stations[1:]: + time.sleep(1) nearby_data = Data(Periods(stations, nearby_station[0])) data.df = self._iterate_over_time(data.df, nearby_data.df, missing_df) missing_df = self._find_missing_data(data.df) From 2cea0b26e16187d8f87be51bc3cb9f3e0b0d22a1 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Sat, 18 May 2024 16:33:55 +0200 Subject: [PATCH 61/67] fix lint --- src/smhi/smhi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/smhi/smhi.py b/src/smhi/smhi.py index ce4faef2..6c1645c8 100644 --- a/src/smhi/smhi.py +++ b/src/smhi/smhi.py @@ -1,7 +1,7 @@ """Read SMHI data.""" -import time import logging +import time from typing import Any, List, Optional, Tuple import pandas as pd @@ -177,9 +177,9 @@ def _iterate_over_time( self, df: pd.DataFrame, nearby_df: pd.DataFrame, missing_df: pd.DataFrame ) -> pd.DataFrame: """Iterate over time.""" - for time, _ in missing_df.iterrows(): - earliertime = df[df.index < time].index.max() - condition = (nearby_df.index > earliertime) & (nearby_df.index < time) + for tajm, _ in missing_df.iterrows(): + earliertime = df[df.index < tajm].index.max() + condition = (nearby_df.index > earliertime) & (nearby_df.index < tajm) if len(nearby_df[condition]) > 0: df = pd.concat([df, nearby_df], axis=0, join="outer") From c674225171f134e6c850cf8fde38f17fc21114d0 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Mon, 20 May 2024 22:32:57 +0200 Subject: [PATCH 62/67] Rewrote sentence --- docs/index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 3cf94d2e..bb9e4653 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,7 +17,8 @@ Currently, only these four APIs are supported ## Examples -You can reach example usage of the different clients +Examples of direct usage of the different clients, and the experimental parent class, +are available here: - [example of Metobs use](/ifk-smhi/metobs-example/) - [example of Mesan use](/ifk-smhi/mesan-example/) From 49f77af4c736c2f124567594bf72dd2997fd5e43 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Mon, 20 May 2024 22:36:13 +0200 Subject: [PATCH 63/67] space after comma --- docs/smhi-example.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/smhi-example.md b/docs/smhi-example.md index 7ea2c32e..db4b2ab3 100644 --- a/docs/smhi-example.md +++ b/docs/smhi-example.md @@ -22,7 +22,7 @@ client.parameters.data #List all available parameters stations = client.get_stations(1) stations.data #List all available stations for parameter 1 -data = client.get_data(1,72630) #Get data from specific station +data = client.get_data(1, 72630) #Get data from specific station #Station 72630 is Gothenburg ``` @@ -46,18 +46,18 @@ from smhi.smhi import SMHI client = SMHI() #Parameter 8 is depth of snow coverage -data = client.get_data(8,180960,40) #Get data from specific station +data = client.get_data(8, 180960, 40) #Get data from specific station #Station 180960 is Kiruna. 40 indicates we will use stations within #a 40km radius to complement any data losses. -data2 = client.get_data(8,180960) #Get comparison data without the +data2 = client.get_data(8, 180960) #Get comparison data without the #interpolation. import matplotlib.pyplot as plt -plt.plot(data.df.index,data.df["Snödjup"]) -plt.plot(data2.df.index,data2.df["Snödjup"]) +plt.plot(data.df.index, data.df["Snödjup"]) +plt.plot(data2.df.index, data2.df["Snödjup"]) plt.ylabel("Snödjup") -plt.legend(["Interpolated (40km)","Station 180960"]) +plt.legend(["Interpolated (40km)", "Station 180960"]) plt.show() ``` @@ -73,7 +73,7 @@ from smhi.smhi import SMHI client = SMHI() #Parameter 8 is depth of snow coverage -data = client.get_data_by_city(8,"Kiruna",40) #Get data from a station +data = client.get_data_by_city(8, "Kiruna", 40) #Get data from a station #close to the city centre of Kiruna, and fill any empty holes with data #from stations within a 40km perimeter From 63e1c8c1c9e22869898f81a714612ec90ca4f003 Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Mon, 20 May 2024 22:38:08 +0200 Subject: [PATCH 64/67] toolbox to package --- docs/smhi-example.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/smhi-example.md b/docs/smhi-example.md index db4b2ab3..928dd775 100644 --- a/docs/smhi-example.md +++ b/docs/smhi-example.md @@ -36,7 +36,7 @@ the actual observational data. See further When there are several stations close to a measurement location and historical data records are incomplete, it is convenient to use nearby stations to fill in the missing -data: This naive interpolation feature is included in the toolbox by the parameter +data: This naive interpolation feature is included in the package by the parameter `radius`. Thus, if we want to fetch historical data on the snow coverage in Kiruna, the call is: @@ -65,7 +65,7 @@ plt.show() There are a lot of stations available: Frequently, we are simply interested in meterological observations from a specific city. Using the library `geopy`, -this is included in the toolbox too: +this is included in the package too: ```python from smhi.smhi import SMHI From 00737319bf787cc3be1f72ba50d8fdd0df55efdf Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Mon, 20 May 2024 22:43:09 +0200 Subject: [PATCH 65/67] spaces in integration test --- tests/integration/test_integration_smhi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_integration_smhi.py b/tests/integration/test_integration_smhi.py index 6ff39d34..9b92a917 100644 --- a/tests/integration/test_integration_smhi.py +++ b/tests/integration/test_integration_smhi.py @@ -10,7 +10,7 @@ class TestIntegrationSMHI: """Integration test of SMHI class.""" @pytest.mark.parametrize( - "parameter, city, period,radius,expected_result", + "parameter, city, period, radius, expected_result", [ ( 1, From 5cf7d2e2f600019f185ad9bc98d26923776d740c Mon Sep 17 00:00:00 2001 From: Anders Nord Date: Wed, 22 May 2024 06:38:07 +0200 Subject: [PATCH 66/67] Add plotly figure --- docs/assets/kiruna_snodjup.html | 7 +++++ docs/smhi-example.md | 54 +++++++++++++++++++++++++++++---- mkdocs.yml | 2 ++ 3 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 docs/assets/kiruna_snodjup.html diff --git a/docs/assets/kiruna_snodjup.html b/docs/assets/kiruna_snodjup.html new file mode 100644 index 00000000..a7a1f815 --- /dev/null +++ b/docs/assets/kiruna_snodjup.html @@ -0,0 +1,7 @@ + + + +
+
+ + \ No newline at end of file diff --git a/docs/smhi-example.md b/docs/smhi-example.md index 928dd775..aebfa7b5 100644 --- a/docs/smhi-example.md +++ b/docs/smhi-example.md @@ -52,15 +52,57 @@ data = client.get_data(8, 180960, 40) #Get data from specific station data2 = client.get_data(8, 180960) #Get comparison data without the #interpolation. +``` + +Visualise the data: +
+ Scatter plot code -import matplotlib.pyplot as plt -plt.plot(data.df.index, data.df["Snödjup"]) -plt.plot(data2.df.index, data2.df["Snödjup"]) -plt.ylabel("Snödjup") -plt.legend(["Interpolated (40km)", "Station 180960"]) -plt.show() +```python +import plotly.graph_objects as go + +d1 = data.df +d2 = data2.df + +index = d1.index.intersection(d2.index) +d2_dropped = d2.drop(index, axis=0) + +fig = go.Figure() +fig.add_trace( + go.Scattergl( + x=d1.index, + y=d1["Snödjup"], + mode="markers", + name="Kiruna station" + ) +) +fig.add_trace( + go.Scattergl( + x=d2_dropped.index, + y=d2_dropped["Snödjup"], + mode="markers", + name="Interpolerat, radie 40 km" + ) +) +fig.update_layout( + title='Historiskt snödjup i Kiruna', + xaxis_title="År", + yaxis_title="Snödjup [m]", + legend={"orientation": "h"}, + margin={"l": 0, "r": 0, "b": 80, "t": 100} +) + +fig.show() ``` +
+ + + ## Finding data from a city There are a lot of stations available: Frequently, we are simply interested diff --git a/mkdocs.yml b/mkdocs.yml index 8515eeb9..05507878 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,12 +11,14 @@ nav: - "Metfcts": metfcts-example.md - "Mesan": mesan-example.md - "Strang": strang-example.md + - "SMHI": smhi-example.md - Reference: - "Clients": - "Metobs reference": metobs-reference.md - "Metfcts reference": metfcts-reference.md - "Mesan reference": mesan-reference.md - "Strang reference": strang-reference.md + - "SMHI reference": smhi-reference.md - "Models": - "Metobs model": metobs-model.md - "Metfcts model": metfcts-model.md From f0a5485938e7650cb72c300f5be00fcffdf1cce6 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Wed, 22 May 2024 18:48:28 +0200 Subject: [PATCH 67/67] This will only work as long as we serve docs on ...se/ifk-smhi/ --- docs/smhi-example.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/smhi-example.md b/docs/smhi-example.md index aebfa7b5..3e246f57 100644 --- a/docs/smhi-example.md +++ b/docs/smhi-example.md @@ -19,7 +19,7 @@ client = SMHI() client.parameters.data #List all available parameters #Parameter 1 is hourly air temperature -stations = client.get_stations(1) +stations = client.get_stations(1) stations.data #List all available stations for parameter 1 data = client.get_data(1, 72630) #Get data from specific station @@ -55,6 +55,7 @@ data2 = client.get_data(8, 180960) #Get comparison data without the ``` Visualise the data: +
Scatter plot code @@ -100,7 +101,7 @@ fig.show() ## Finding data from a city