From 0de96a301ec518d41124d92694f018e89e33a631 Mon Sep 17 00:00:00 2001 From: Mladen Gibanica <11275336+mgcth@users.noreply.github.com> Date: Sat, 30 Mar 2024 18:23:05 +0100 Subject: [PATCH] Raise on request error instead of empty data --- src/smhi/metobs.py | 126 +++++-------------- src/smhi/strang.py | 11 +- src/smhi/utils.py | 7 +- tests/integration/test_integration_metobs.py | 10 +- tests/unit/test_unit_metobs.py | 24 ++-- 5 files changed, 59 insertions(+), 119 deletions(-) diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index c35231de..155e0b80 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -31,9 +31,7 @@ def __init__(self) -> None: self.summary: Optional[str] = None self.link: Optional[str] = None - def _get_and_parse_request( - self, url: str, model: MetobsModels - ) -> Optional[MetobsModels]: + def _get_and_parse_request(self, url: str, model: MetobsModels) -> MetobsModels: """Get and parse API request. Only JSON supported. Args: @@ -44,14 +42,9 @@ def _get_and_parse_request( pydantic model """ response = get_request(url) - - self.headers = response.headers - - if response.ok is not True: - return None - model = model.model_validate_json(response.content) + self.headers = response.headers self.key = model.key self.updated = model.updated self.title = model.title @@ -62,7 +55,7 @@ def _get_and_parse_request( def _get_url( self, - data: Optional[list[Any]], + data: list[Any], key: str, parameter: Union[str, int], data_type: str = "json", @@ -86,9 +79,6 @@ def _get_url( lambda: "application/json", json="application/json" )[data_type] - if data is None: - raise ValueError("No data to iterate over.") - try: requested_data = [x for x in data if getattr(x, key) == str(parameter)][0] url = [x.href for x in requested_data.link if x.type == self.data_type][0] @@ -104,7 +94,6 @@ class Versions(BaseMetobs): """Get available versions of Metobs API.""" _base_url: str = "https://opendata-download-metobs.smhi.se/api.{data_type}" - data = None def __init__( self, @@ -129,16 +118,12 @@ def __init__( self._base_url.format(data_type=data_type), MetobsVersionModel ) - if model is not None: - self.data = model.data + self.data = model.data class Parameters(BaseMetobs): """Get parameters for version 1 of Metobs API.""" - resource = None - data = None - def __init__( self, versions_object: Optional[Versions] = None, @@ -155,7 +140,6 @@ def __init__( Raises: TypeError: data_type not supported NotImplementedError: version not implemented - ValueError """ super().__init__() @@ -169,28 +153,19 @@ def __init__( if versions_object is None: versions_object = Versions() - if versions_object.data is None: - raise ValueError("No data to iterate over.") - url, _ = self._get_url(versions_object.data, "key", version, data_type) model = self._get_and_parse_request(url, MetobsParameterModel) self.versions_object = versions_object self.selected_version = version - if model is not None: - self.resource = model.resource - self.data = model.data + self.resource = model.resource + self.data = model.data class Stations(BaseMetobs): """Get stations from parameter for version 1 of Metobs API.""" - value_type = None - station_set = None - station = None - data = None - def __init__( self, parameters_in_version: Parameters, @@ -209,14 +184,10 @@ def __init__( Raises: TypeError: data_type not supported NotImplementedError: parameter not implemented - ValueError """ super().__init__() self.selected_parameter: Optional[Union[int, str]] = None - if parameters_in_version.data is None: - raise ValueError("No data to iterate over.") - if data_type != "json": raise TypeError("Only json supported.") @@ -241,11 +212,10 @@ def __init__( self.parameters_in_version = parameters_in_version - if model is not None: - self.value_type = model.value_type - self.station_set = model.station_set - self.station = model.station - self.data = model.data + self.value_type = model.value_type + self.station_set = model.station_set + self.station = model.station + self.data = model.data class Periods(BaseMetobs): @@ -254,16 +224,6 @@ class Periods(BaseMetobs): Note that stationset_title is not supported """ - owner = None - owner_category = None - measuring_stations = None - active = None - from_ = None - to = None - position = None - period = None - data = None - def __init__( self, stations_in_parameter: Stations, @@ -288,9 +248,6 @@ def __init__( super().__init__() self.selected_station: Optional[Union[int, str]] = None - if stations_in_parameter.data is None: - raise ValueError("No data to iterate over.") - if data_type != "json": raise TypeError("Only json supported.") @@ -320,16 +277,15 @@ def __init__( self.stations_in_parameter = stations_in_parameter - if model is not None: - self.owner = model.owner - self.owner_category = model.owner_category - self.measuring_stations = model.measuring_stations - self.active = model.active - self.from_ = model.from_ - self.to = model.to - self.position = model.position - self.period = model.period - self.data = model.data + self.owner = model.owner + self.owner_category = model.owner_category + self.measuring_stations = model.measuring_stations + self.active = model.active + self.from_ = model.from_ + self.to = model.to + self.position = model.position + self.period = model.period + self.data = model.data class Data(BaseMetobs): @@ -340,13 +296,6 @@ class Data(BaseMetobs): _metobs_parameter_dygn: List[str] = ["Representativt dygn"] _metobs_parameter_manad: List[str] = ["Representativ månad"] - from_ = None - to = None - station = None - parameter = None - period = None - data = None - def __init__( self, periods_in_station: Periods, @@ -364,13 +313,9 @@ def __init__( Raises: TypeError: data_type not supported NotImplementedError: period not implemented - ValueError: No data to iterate over """ super().__init__() - if periods_in_station.data is None: - raise ValueError("No data to iterate over.") - if data_type != "json": raise TypeError("Only json supported.") @@ -393,24 +338,21 @@ def __init__( url, _ = self._get_url(periods_in_station.period, "key", period, data_type) model = self._get_and_parse_request(url, MetobsData) - if model is not None: - data_model = self._get_data(model.data) - stationdata = data_model.stationdata - stationdata = self._clean_columns(stationdata) - stationdata = self._drop_nan(stationdata) - - if ( - self._has_datetime_columns(stationdata) is True - and not stationdata.empty - ): - stationdata = self._set_dataframe_index(stationdata) - - self.from_ = model.from_ - self.to = model.to - self.station = data_model.station - self.parameter = data_model.parameter - self.period = data_model.period - self.data = stationdata + data_model = self._get_data(model.data) + stationdata = data_model.stationdata + stationdata = self._clean_columns(stationdata) + stationdata = self._drop_nan(stationdata) + + if self._has_datetime_columns(stationdata) is True and not stationdata.empty: + stationdata = self._set_dataframe_index(stationdata) + + self.from_ = model.from_ + self.to = model.to + + self.station = data_model.station + self.parameter = data_model.parameter + self.period = data_model.period + self.data = stationdata def _check_available_periods(self, data: Tuple[Optional[str]], period: str) -> bool: """Check available periods. diff --git a/src/smhi/strang.py b/src/smhi/strang.py index 3edfec4a..4b48fc6d 100644 --- a/src/smhi/strang.py +++ b/src/smhi/strang.py @@ -110,8 +110,8 @@ def get_point( raw_url = self._point_raw_url url = self._build_base_point_url(raw_url, strang_parameter, longitude, latitude) - url = self._build_time_point_url(url + "1", time_from, time_to, time_interval) - data, header, status = self._get_and_load_data(url, RequestType["POINT"]) + url = self._build_time_point_url(url, time_from, time_to, time_interval) + data, header, status = self._get_and_load_data(url + "1", RequestType["POINT"]) return StrangPoint( parameter_key=strang_parameter.key, @@ -277,7 +277,7 @@ def _build_time_multipoint_url(self, url: str, time_interval: Optional[str]) -> def _get_and_load_data( self, url: str, request: RequestType - ) -> tuple[Optional[pd.DataFrame], CaseInsensitiveDict[str], int]: + ) -> tuple[pd.DataFrame, CaseInsensitiveDict[str], int]: """Fetch requested point data and parse it with datetime. Args: @@ -289,11 +289,8 @@ def _get_and_load_data( status code """ response = get_request(url) - - if response.ok is not True: - return None, response.headers, response.status_code - data = df = json.loads(response.content) + if request == RequestType.POINT: df = self._parse_point_data(data) else: diff --git a/src/smhi/utils.py b/src/smhi/utils.py index 30f37c39..5d50f4c0 100644 --- a/src/smhi/utils.py +++ b/src/smhi/utils.py @@ -21,11 +21,12 @@ def get_request(url: str) -> requests.Response: requests.exceptions.HTTPError """ logger.debug(f"Fetching from {url}.") + response = requests.get(url, timeout=200) if response.status_code != STATUS_OK: - logger.warning(f"Request failed for {url}.") - else: - logger.debug(f"Successful request from {url}.") + raise requests.exceptions.HTTPError(f"Could request from {url}.") + + logger.debug(f"Successful request from {url}.") return response diff --git a/tests/integration/test_integration_metobs.py b/tests/integration/test_integration_metobs.py index c1e038be..11b8dfff 100644 --- a/tests/integration/test_integration_metobs.py +++ b/tests/integration/test_integration_metobs.py @@ -6,7 +6,7 @@ import pandas as pd import pytest from smhi.metobs import Data, Parameters, Periods, Stations -from smhi.models.metobs_parameters import ParameterItem +from smhi.models.metobs_parameters import MetobsParameterItem METOBS_INTEGRATION = {} for i in [1, 2, 22]: @@ -34,7 +34,7 @@ class TestIntegrationMetobs: "metobs", "Meteorologiska observationer från SMHI: Välj " + "version (sedan parameter, station och tidsutsnitt)", - ParameterItem( + MetobsParameterItem( key="1", title="Lufttemperatur", summary="momentanvärde, 1 gång/tim", @@ -56,7 +56,7 @@ class TestIntegrationMetobs: "metobs", "Meteorologiska observationer från SMHI: Välj " + "version (sedan parameter, station och tidsutsnitt)", - ParameterItem( + MetobsParameterItem( key="1", title="Lufttemperatur", summary="momentanvärde, 1 gång/tim", @@ -78,7 +78,7 @@ class TestIntegrationMetobs: "metobs", "Meteorologiska observationer från SMHI: Välj " + "version (sedan parameter, station och tidsutsnitt)", - ParameterItem( + MetobsParameterItem( key="2", title="Lufttemperatur", summary="medelvärde 1 dygn, 1 gång/dygn, kl 00", @@ -100,7 +100,7 @@ class TestIntegrationMetobs: "metobs", "Meteorologiska observationer från SMHI: Välj " + "version (sedan parameter, station och tidsutsnitt)", - ParameterItem( + MetobsParameterItem( key="22", title="Lufttemperatur", summary="medel, 1 gång per månad", diff --git a/tests/unit/test_unit_metobs.py b/tests/unit/test_unit_metobs.py index 8b0cb512..74f0798b 100644 --- a/tests/unit/test_unit_metobs.py +++ b/tests/unit/test_unit_metobs.py @@ -15,11 +15,11 @@ Stations, Versions, ) -from smhi.models.metobs_data import DataModel -from smhi.models.metobs_parameters import ParameterItem, ParameterModel -from smhi.models.metobs_periods import PeriodModel -from smhi.models.metobs_stations import StationModel -from smhi.models.metobs_versions import VersionModel +from smhi.models.metobs_data import MetobsData +from smhi.models.metobs_parameters import MetobsParameterItem, MetobsParameterModel +from smhi.models.metobs_periods import MetobsPeriodModel +from smhi.models.metobs_stations import MetobsStationModel +from smhi.models.metobs_versions import MetobsVersionModel class MockModelInner(BaseModel): @@ -102,7 +102,7 @@ def get_data(file, load_type=None): file_contents = tuple([tuple(x) for x in json.load(f)]) elif load_type == "parameters": file_contents = tuple( - ParameterItem.model_validate_json(x) for x in json.load(f) + MetobsParameterItem.model_validate_json(x) for x in json.load(f) ) elif load_type == "data": file_contents = MockResponse(200, None, f.read().encode("utf-8")) @@ -121,7 +121,7 @@ def setup_versions(): expected answer as pydantic structure """ mocked_response = get_response("tests/fixtures/metobs/versions.txt") - mocked_model = VersionModel.model_validate_json(mocked_response.content) + mocked_model = MetobsVersionModel.model_validate_json(mocked_response.content) return mocked_response, mocked_model @@ -138,7 +138,7 @@ def setup_parameters(setup_versions): _, mocked_model_versions = setup_versions mocked_response = get_response("tests/fixtures/metobs/parameters.txt") - mocked_model = ParameterModel.model_validate_json(mocked_response.content) + mocked_model = MetobsParameterModel.model_validate_json(mocked_response.content) mocked_data = get_data("tests/fixtures/metobs/parameters_data.json", "parameters") return ( @@ -161,7 +161,7 @@ def setup_stations(setup_parameters): _, mocked_model_parameters, _, _ = setup_parameters mocked_response = get_response("tests/fixtures/metobs/stations.txt") - mocked_model = StationModel.model_validate_json(mocked_response.content) + mocked_model = MetobsStationModel.model_validate_json(mocked_response.content) mocked_data = get_data("tests/fixtures/metobs/stations_data.json") return ( @@ -184,7 +184,7 @@ def setup_periods(setup_stations): _, mocked_model_stations, _, _ = setup_stations mocked_response = get_response("tests/fixtures/metobs/periods.txt") - mocked_model = PeriodModel.model_validate_json(mocked_response.content) + mocked_model = MetobsPeriodModel.model_validate_json(mocked_response.content) mocked_data = get_data("tests/fixtures/metobs/periods_data.json", "period") return ( @@ -207,7 +207,7 @@ def setup_periods_set(setup_stations): _, mocked_model_stations, _, _ = setup_stations mocked_response = get_response("tests/fixtures/metobs/periods_set.txt") - mocked_model = PeriodModel.model_validate_json(mocked_response.content) + mocked_model = MetobsPeriodModel.model_validate_json(mocked_response.content) mocked_data = get_data("tests/fixtures/metobs/periods_data_set.json", "period") return ( @@ -230,7 +230,7 @@ def setup_data(setup_periods): _, mocked_model_periods, _, _ = setup_periods mocked_response = get_response("tests/fixtures/metobs/data.txt", encode=True) - mocked_model = DataModel.model_validate_json(mocked_response.content) + mocked_model = MetobsData.model_validate_json(mocked_response.content) mocked_csv_data = get_data("tests/fixtures/metobs/data_csv.csv", "data") mocked_station = get_data("tests/fixtures/metobs/data_station.json")