diff --git a/src/smhi/metobs.py b/src/smhi/metobs.py index c1a5372d..c35231de 100644 --- a/src/smhi/metobs.py +++ b/src/smhi/metobs.py @@ -3,16 +3,16 @@ import io import logging from collections import defaultdict -from typing import Any, Optional, Union +from typing import Any, Dict, List, Optional, Tuple, Union import pandas as pd from requests.structures import CaseInsensitiveDict from smhi.constants import METOBS_AVAILABLE_PERIODS -from smhi.models.metobs_data import DataModel, Datum, MetobsData -from smhi.models.metobs_parameters import ParameterModel -from smhi.models.metobs_periods import PeriodModel -from smhi.models.metobs_stations import StationModel -from smhi.models.metobs_versions import VersionModel +from smhi.models.metobs_data import MetobsData, MetobsDataModel, MetobsDatum +from smhi.models.metobs_parameters import MetobsParameterModel +from smhi.models.metobs_periods import MetobsPeriodModel +from smhi.models.metobs_stations import MetobsStationModel +from smhi.models.metobs_versions import MetobsVersionModel from smhi.models.variable_model import MetobsModels from smhi.utils import get_request @@ -31,7 +31,9 @@ def __init__(self) -> None: self.summary: Optional[str] = None self.link: Optional[str] = None - def _get_and_parse_request(self, url: str, model: MetobsModels) -> MetobsModels: + def _get_and_parse_request( + self, url: str, model: MetobsModels + ) -> Optional[MetobsModels]: """Get and parse API request. Only JSON supported. Args: @@ -40,14 +42,16 @@ def _get_and_parse_request(self, url: str, model: MetobsModels) -> MetobsModels: Returns: pydantic model - - Raise: - requests.exceptions.HTTPError """ response = get_request(url) - model = model.model_validate_json(response.content) self.headers = response.headers + + if response.ok is not True: + return None + + model = model.model_validate_json(response.content) + self.key = model.key self.updated = model.updated self.title = model.title @@ -58,7 +62,7 @@ def _get_and_parse_request(self, url: str, model: MetobsModels) -> MetobsModels: def _get_url( self, - data: list[Any], + data: Optional[list[Any]], key: str, parameter: Union[str, int], data_type: str = "json", @@ -81,6 +85,10 @@ def _get_url( self.data_type = defaultdict( lambda: "application/json", json="application/json" )[data_type] + + if data is None: + raise ValueError("No data to iterate over.") + try: requested_data = [x for x in data if getattr(x, key) == str(parameter)][0] url = [x.href for x in requested_data.link if x.type == self.data_type][0] @@ -95,7 +103,8 @@ def _get_url( class Versions(BaseMetobs): """Get available versions of Metobs API.""" - _base_url = "https://opendata-download-metobs.smhi.se/api.{data_type}" + _base_url: str = "https://opendata-download-metobs.smhi.se/api.{data_type}" + data = None def __init__( self, @@ -117,15 +126,19 @@ def __init__( raise TypeError("Only json supported.") model = self._get_and_parse_request( - self._base_url.format(data_type=data_type), VersionModel + self._base_url.format(data_type=data_type), MetobsVersionModel ) - self.data = model.version + if model is not None: + self.data = model.data class Parameters(BaseMetobs): """Get parameters for version 1 of Metobs API.""" + resource = None + data = None + def __init__( self, versions_object: Optional[Versions] = None, @@ -142,6 +155,7 @@ def __init__( Raises: TypeError: data_type not supported NotImplementedError: version not implemented + ValueError """ super().__init__() @@ -155,19 +169,28 @@ def __init__( if versions_object is None: versions_object = Versions() + if versions_object.data is None: + raise ValueError("No data to iterate over.") + url, _ = self._get_url(versions_object.data, "key", version, data_type) - model = self._get_and_parse_request(url, ParameterModel) + model = self._get_and_parse_request(url, MetobsParameterModel) self.versions_object = versions_object self.selected_version = version - self.resource = model.resource - self.data = model.data + if model is not None: + self.resource = model.resource + self.data = model.data class Stations(BaseMetobs): """Get stations from parameter for version 1 of Metobs API.""" + value_type = None + station_set = None + station = None + data = None + def __init__( self, parameters_in_version: Parameters, @@ -186,10 +209,14 @@ def __init__( Raises: TypeError: data_type not supported NotImplementedError: parameter not implemented + ValueError """ super().__init__() self.selected_parameter: Optional[Union[int, str]] = None + if parameters_in_version.data is None: + raise ValueError("No data to iterate over.") + if data_type != "json": raise TypeError("Only json supported.") @@ -210,14 +237,15 @@ def __init__( parameters_in_version.resource, "title", parameter_title, data_type ) - model = self._get_and_parse_request(url, StationModel) + model = self._get_and_parse_request(url, MetobsStationModel) self.parameters_in_version = parameters_in_version - self.value_type = model.value_type - self.station_set = model.station_set - self.station = model.station - self.data = model.data + if model is not None: + self.value_type = model.value_type + self.station_set = model.station_set + self.station = model.station + self.data = model.data class Periods(BaseMetobs): @@ -226,6 +254,16 @@ class Periods(BaseMetobs): Note that stationset_title is not supported """ + owner = None + owner_category = None + measuring_stations = None + active = None + from_ = None + to = None + position = None + period = None + data = None + def __init__( self, stations_in_parameter: Stations, @@ -250,6 +288,9 @@ def __init__( super().__init__() self.selected_station: Optional[Union[int, str]] = None + if stations_in_parameter.data is None: + raise ValueError("No data to iterate over.") + if data_type != "json": raise TypeError("Only json supported.") @@ -275,28 +316,36 @@ def __init__( stations_in_parameter.station_set, "key", station_set, data_type ) - model = self._get_and_parse_request(url, PeriodModel) + model = self._get_and_parse_request(url, MetobsPeriodModel) self.stations_in_parameter = stations_in_parameter - self.owner = model.owner - self.owner_category = model.owner_category - self.measuring_stations = model.measuring_stations - self.active = model.active - self.from_ = model.from_ - self.to = model.to - self.position = model.position - self.period = model.period - self.data = model.data + if model is not None: + self.owner = model.owner + self.owner_category = model.owner_category + self.measuring_stations = model.measuring_stations + self.active = model.active + self.from_ = model.from_ + self.to = model.to + self.position = model.position + self.period = model.period + self.data = model.data class Data(BaseMetobs): """Get data from period for version 1 of Metobs API.""" - _metobs_available_periods = METOBS_AVAILABLE_PERIODS - _metobs_parameter_tim = ["Datum", "Tid (UTC)"] - _metobs_parameter_dygn = ["Representativt dygn"] - _metobs_parameter_manad = ["Representativ månad"] + _metobs_available_periods: Dict[str, str] = METOBS_AVAILABLE_PERIODS + _metobs_parameter_tim: List[str] = ["Datum", "Tid (UTC)"] + _metobs_parameter_dygn: List[str] = ["Representativt dygn"] + _metobs_parameter_manad: List[str] = ["Representativ månad"] + + from_ = None + to = None + station = None + parameter = None + period = None + data = None def __init__( self, @@ -315,13 +364,17 @@ def __init__( Raises: TypeError: data_type not supported NotImplementedError: period not implemented + ValueError: No data to iterate over """ super().__init__() + if periods_in_station.data is None: + raise ValueError("No data to iterate over.") + if data_type != "json": raise TypeError("Only json supported.") - if self._check_available_periods(periods_in_station, period): + if self._check_available_periods(periods_in_station.data, period): logger.info( "Found only one period to download. " + f"Overriding the user selected {period} with the found {periods_in_station.data[0]}." @@ -338,27 +391,28 @@ def __init__( self.selected_period = period url, _ = self._get_url(periods_in_station.period, "key", period, data_type) - model = self._get_and_parse_request(url, DataModel) - - self.from_ = model.from_ - self.to = model.to - - data_model = self._get_data(model.data) - stationdata = data_model.stationdata - stationdata = self._clean_columns(stationdata) - stationdata = self._drop_nan(stationdata) - - if self._has_datetime_columns(stationdata) is True and not stationdata.empty: - stationdata = self._set_dataframe_index(stationdata) - - self.station = data_model.station - self.parameter = data_model.parameter - self.period = data_model.period - self.data = stationdata - - def _check_available_periods( - self, periods_in_station: Periods, period: str - ) -> bool: + model = self._get_and_parse_request(url, MetobsData) + + if model is not None: + data_model = self._get_data(model.data) + stationdata = data_model.stationdata + stationdata = self._clean_columns(stationdata) + stationdata = self._drop_nan(stationdata) + + if ( + self._has_datetime_columns(stationdata) is True + and not stationdata.empty + ): + stationdata = self._set_dataframe_index(stationdata) + + self.from_ = model.from_ + self.to = model.to + self.station = data_model.station + self.parameter = data_model.parameter + self.period = data_model.period + self.data = stationdata + + def _check_available_periods(self, data: Tuple[Optional[str]], period: str) -> bool: """Check available periods. Args: @@ -366,16 +420,18 @@ def _check_available_periods( period: select period from: latest-hour, latest-day, latest-months or corrected-archive - Returns + Returns: boolean """ return ( - len(periods_in_station.data) == 1 - and periods_in_station.data[0] != period + len(data) == 1 + and data[0] != period and period in self._metobs_available_periods ) - def _get_data(self, raw_data: list[Datum], type: str = "text/plain") -> MetobsData: + def _get_data( + self, raw_data: list[MetobsDatum], type: str = "text/plain" + ) -> MetobsDataModel: """Get the selected data file. Args: @@ -400,12 +456,12 @@ def _get_data(self, raw_data: list[Datum], type: str = "text/plain") -> MetobsDa # these are the two cases I've found. Generalise if there are others if len(csv_content) == 2: - data_model = MetobsData( + data_model = MetobsDataModel( parameter=self._parse_csv(csv_content[0]), stationdata=self._parse_csv(csv_content[1]), ) else: - data_model = MetobsData( + data_model = MetobsDataModel( station=self._parse_csv(csv_content[0]), parameter=self._parse_csv(csv_content[1]), period=self._parse_csv(csv_content[2]), diff --git a/src/smhi/models/metobs_data.py b/src/smhi/models/metobs_data.py index 9bd205e0..cadebdc1 100644 --- a/src/smhi/models/metobs_data.py +++ b/src/smhi/models/metobs_data.py @@ -10,32 +10,32 @@ from pydantic import BaseModel, ConfigDict, Field -class LinkItem(BaseModel): +class MetobsLinkItem(BaseModel): href: str rel: str type: str -class Datum(BaseModel): +class MetobsDatum(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str summary: str - link: List[LinkItem] + link: List[MetobsLinkItem] -class DataModel(BaseModel): +class MetobsData(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str summary: str from_: int = Field(..., alias="from") to: int - link: List[LinkItem] - data: List[Datum] + link: List[MetobsLinkItem] + data: List[MetobsDatum] -class MetobsData(BaseModel): +class MetobsDataModel(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) station: Optional[pd.DataFrame] = None diff --git a/src/smhi/models/metobs_parameters.py b/src/smhi/models/metobs_parameters.py index 35226660..d06d0f0b 100644 --- a/src/smhi/models/metobs_parameters.py +++ b/src/smhi/models/metobs_parameters.py @@ -9,52 +9,54 @@ from pydantic import BaseModel, Field, field_validator -class ParameterItem(BaseModel): +class MetobsParameterItem(BaseModel): key: Optional[str] title: str summary: str unit: str -class LinkItem(BaseModel): +class MetobsParameterLinkItem(BaseModel): href: str rel: str type: str -class GeoBox(BaseModel): +class MetobsParameterGeoBox(BaseModel): min_latitude: float = Field(..., alias="minLatitude") min_longitude: float = Field(..., alias="minLongitude") max_latitude: float = Field(..., alias="maxLatitude") max_longitude: float = Field(..., alias="maxLongitude") -class ResourceItem(BaseModel): +class MetobsParameterResourceItem(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str summary: str - link: List[LinkItem] + link: List[MetobsParameterLinkItem] unit: str - geo_box: GeoBox = Field(..., alias="geoBox") + geo_box: MetobsParameterGeoBox = Field(..., alias="geoBox") -class ParameterModel(BaseModel): +class MetobsParameterModel(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str summary: str - link: List[LinkItem] - resource: List[ResourceItem] + link: List[MetobsParameterLinkItem] + resource: List[MetobsParameterResourceItem] @field_validator("resource") @classmethod - def serialise_resource_in_order(cls, resource: List[ResourceItem]): + def serialise_resource_in_order(cls, resource: List[MetobsParameterResourceItem]): return sorted(resource, key=lambda x: int(x.key)) @property - def data(self) -> Tuple[ParameterItem, ...]: + def data(self) -> Tuple[MetobsParameterItem, ...]: return tuple( - ParameterItem(key=x.key, title=x.title, summary=x.summary, unit=x.unit) + MetobsParameterItem( + key=x.key, title=x.title, summary=x.summary, unit=x.unit + ) for x in self.resource ) diff --git a/src/smhi/models/metobs_periods.py b/src/smhi/models/metobs_periods.py index a6bd148f..e65801be 100644 --- a/src/smhi/models/metobs_periods.py +++ b/src/smhi/models/metobs_periods.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field, field_validator -class PositionItem(BaseModel): +class MetobsPositionItem(BaseModel): from_: int = Field(..., alias="from") to: int height: float @@ -17,21 +17,21 @@ class PositionItem(BaseModel): longitude: float -class LinkItem(BaseModel): +class MetobsLinkItem(BaseModel): href: str rel: str type: str -class PeriodItem(BaseModel): +class MetobsPeriodItem(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str summary: str - link: List[LinkItem] + link: List[MetobsLinkItem] -class PeriodModel(BaseModel): +class MetobsPeriodModel(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str @@ -42,13 +42,13 @@ class PeriodModel(BaseModel): summary: str from_: Optional[int] = Field(default=None, alias="from") to: Optional[int] = None - position: Optional[List[PositionItem]] = None - link: List[LinkItem] - period: List[PeriodItem] + position: Optional[List[MetobsPositionItem]] = None + link: List[MetobsLinkItem] + period: List[MetobsPeriodItem] @field_validator("period") @classmethod - def serialise_period_in_order(cls, period: List[PeriodItem]): + def serialise_period_in_order(cls, period: List[MetobsPeriodItem]): return sorted(period, key=lambda x: x.key) @property diff --git a/src/smhi/models/metobs_stations.py b/src/smhi/models/metobs_stations.py index 112b9b5e..3f7f1990 100644 --- a/src/smhi/models/metobs_stations.py +++ b/src/smhi/models/metobs_stations.py @@ -9,26 +9,26 @@ from pydantic import BaseModel, Field, field_validator -class LinkItem(BaseModel): +class MetobsStationLinkItem(BaseModel): href: str rel: str type: str -class StationSetItem(BaseModel): +class MetobsStationSetItem(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str summary: str - link: List[LinkItem] + link: List[MetobsStationLinkItem] -class StationItem(BaseModel): +class MetobsStationItem(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str summary: str - link: List[LinkItem] + link: List[MetobsStationLinkItem] name: str owner: str owner_category: str = Field(..., alias="ownerCategory") @@ -42,20 +42,20 @@ class StationItem(BaseModel): to: int -class StationModel(BaseModel): +class MetobsStationModel(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str summary: str unit: str value_type: str = Field(..., alias="valueType") - link: List[LinkItem] - station_set: List[StationSetItem] = Field(..., alias="stationSet") - station: List[StationItem] + link: List[MetobsStationLinkItem] + station_set: List[MetobsStationSetItem] = Field(..., alias="stationSet") + station: List[MetobsStationItem] @field_validator("station") @classmethod - def serialise_station_in_order(cls, station: List[StationItem]): + def serialise_station_in_order(cls, station: List[MetobsStationItem]): return sorted(station, key=lambda x: int(x.id)) @property diff --git a/src/smhi/models/metobs_versions.py b/src/smhi/models/metobs_versions.py index b88b35ac..33cd057a 100644 --- a/src/smhi/models/metobs_versions.py +++ b/src/smhi/models/metobs_versions.py @@ -9,28 +9,28 @@ from pydantic import BaseModel -class LinkItem(BaseModel): +class MetobsVersionLinkItem(BaseModel): href: str rel: str type: str -class VersionItem(BaseModel): +class MetobsVersionItem(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str summary: str - link: List[LinkItem] + link: List[MetobsVersionLinkItem] -class VersionModel(BaseModel): +class MetobsVersionModel(BaseModel): key: Optional[str] = None updated: Optional[int] = None title: str summary: str - link: List[LinkItem] - version: List[VersionItem] + link: List[MetobsVersionLinkItem] + version: List[MetobsVersionItem] @property - def data(self) -> List[VersionItem]: + def data(self) -> List[MetobsVersionItem]: return self.version diff --git a/src/smhi/models/strang_model.py b/src/smhi/models/strang_model.py index a370fee9..9013f4a2 100644 --- a/src/smhi/models/strang_model.py +++ b/src/smhi/models/strang_model.py @@ -38,12 +38,12 @@ class StrangMultiPointItem(BaseModel): class StrangMultiPoint(BaseModel): parameter_key: int parameter_meaning: str - valid_time: Optional[str] = None - time_interval: Optional[str] = None + valid_time: Optional[str] + time_interval: Optional[str] url: str status: int headers: Dict[str, str] - data: DataFrame[StrangMultiPointSchema] + data: Optional[DataFrame[StrangMultiPointSchema]] class StrangPointItem(BaseModel): @@ -56,10 +56,10 @@ class StrangPoint(BaseModel): parameter_meaning: str longitude: float latitude: float - time_from: Optional[str] = None - time_to: Optional[str] = None - time_interval: Optional[str] = None + time_from: Optional[str] + time_to: Optional[str] + time_interval: Optional[str] url: str status: int headers: Dict[str, str] - data: DataFrame[StrangPointSchema] + data: Optional[DataFrame[StrangPointSchema]] diff --git a/src/smhi/models/variable_model.py b/src/smhi/models/variable_model.py index d2a17be9..db36413a 100644 --- a/src/smhi/models/variable_model.py +++ b/src/smhi/models/variable_model.py @@ -18,14 +18,19 @@ MetfctsPolygon, MetfctsValidTime, ) -from smhi.models.metobs_data import DataModel -from smhi.models.metobs_parameters import ParameterModel -from smhi.models.metobs_periods import PeriodModel -from smhi.models.metobs_stations import StationModel -from smhi.models.metobs_versions import VersionModel +from smhi.models.metobs_data import MetobsDataModel +from smhi.models.metobs_parameters import MetobsParameterModel +from smhi.models.metobs_periods import MetobsPeriodModel +from smhi.models.metobs_stations import MetobsStationModel +from smhi.models.metobs_versions import MetobsVersionModel MetobsModels = TypeVar( - "MetobsModels", VersionModel, ParameterModel, StationModel, PeriodModel, DataModel + "MetobsModels", + MetobsVersionModel, + MetobsParameterModel, + MetobsStationModel, + MetobsPeriodModel, + MetobsDataModel, ) Parameters = TypeVar("Parameters", MesanParameters, MetfctsParameters) diff --git a/src/smhi/strang.py b/src/smhi/strang.py index c2de28c8..3edfec4a 100644 --- a/src/smhi/strang.py +++ b/src/smhi/strang.py @@ -6,8 +6,9 @@ import json import logging from collections import defaultdict +from enum import Enum from functools import partial -from typing import Optional +from typing import Any, Optional import arrow import pandas as pd @@ -24,6 +25,11 @@ logger = logging.getLogger(__name__) +class RequestType(Enum): + POINT = 1 + MULTIPOINT = 2 + + class Strang: """SMHI Strang class. Only supports category strang1g and version 1.""" @@ -104,8 +110,8 @@ def get_point( raw_url = self._point_raw_url url = self._build_base_point_url(raw_url, strang_parameter, longitude, latitude) - url = self._build_time_point_url(url, time_from, time_to, time_interval) - data, header, status = self._get_and_load_data(url, strang_parameter) + url = self._build_time_point_url(url + "1", time_from, time_to, time_interval) + data, header, status = self._get_and_load_data(url, RequestType["POINT"]) return StrangPoint( parameter_key=strang_parameter.key, @@ -155,7 +161,7 @@ def get_multipoint( raw_url = self._multipoint_raw_url url = self._build_base_multipoint_url(raw_url, strang_parameter, valid_time) url = self._build_time_multipoint_url(url, time_interval) - data, header, status = self._get_and_load_data(url, strang_parameter) + data, header, status = self._get_and_load_data(url, RequestType["MULTIPOINT"]) return StrangMultiPoint( parameter_key=strang_parameter.key, @@ -186,11 +192,7 @@ def _build_base_point_url( Returns: formatted url string """ - return url( - lon=longitude, - lat=latitude, - parameter=parameter.key, - ) + return url(lon=longitude, lat=latitude, parameter=parameter.key) def _build_base_multipoint_url( self, url: partial[str], parameter: StrangParameter, valid_time: str @@ -206,10 +208,7 @@ def _build_base_multipoint_url( Returns: formatted url string """ - return url( - validtime=valid_time, - parameter=parameter.key, - ) + return url(validtime=valid_time, parameter=parameter.key) def _build_time_point_url( self, @@ -277,13 +276,12 @@ def _build_time_multipoint_url(self, url: str, time_interval: Optional[str]) -> return url def _get_and_load_data( - self, url: str, parameter: StrangParameter - ) -> tuple[pd.DataFrame, CaseInsensitiveDict[str], int]: + self, url: str, request: RequestType + ) -> tuple[Optional[pd.DataFrame], CaseInsensitiveDict[str], int]: """Fetch requested point data and parse it with datetime. Args: url: url to fetch from - parameter: strang parameter Returns: data @@ -291,14 +289,17 @@ def _get_and_load_data( status code """ response = get_request(url) - data = json.loads(response.content) - if "date_time" in data[0]: - data = self._parse_point_data(data, parameter) + if response.ok is not True: + return None, response.headers, response.status_code + + data = df = json.loads(response.content) + if request == RequestType.POINT: + df = self._parse_point_data(data) else: - data = self._parse_multipoint_data(data, parameter) + df = self._parse_multipoint_data(data) - return data, response.headers, response.status_code + return df, response.headers, response.status_code def _parse_datetime( self, date_time: Optional[str], parameter: StrangParameter @@ -332,15 +333,16 @@ def _parse_datetime( ) ) - def _parse_point_data(self, data: list, parameter: StrangParameter) -> pd.DataFrame: + def _parse_point_data( + self, data: list[CaseInsensitiveDict[Any]] + ) -> Optional[pd.DataFrame]: """Parse point data into a pandas DataFrame. Args: - data: data as a list - parameter: strang parameter + url: url of request Returns - data_pd: pandas dataframe + pandas dataframe """ for entry in data: entry["date_time"] = arrow.get(entry["date_time"]).datetime @@ -351,13 +353,12 @@ def _parse_point_data(self, data: list, parameter: StrangParameter) -> pd.DataFr return data_pd def _parse_multipoint_data( - self, data: list, parameter: StrangParameter + self, data: list[CaseInsensitiveDict[Any]] ) -> pd.DataFrame: """Parse multipoint data into a pandas DataFrame. Args: data: data as a list - parameter: strang parameter Returns data_pd: pandas dataframe diff --git a/src/smhi/utils.py b/src/smhi/utils.py index dbbf9379..30f37c39 100644 --- a/src/smhi/utils.py +++ b/src/smhi/utils.py @@ -20,13 +20,12 @@ def get_request(url: str) -> requests.Response: Raises: requests.exceptions.HTTPError """ - logger.info(f"Fetching from {url}.") - + logger.debug(f"Fetching from {url}.") response = requests.get(url, timeout=200) if response.status_code != STATUS_OK: - raise requests.exceptions.HTTPError(f"Could not load from given URL: {url}.") - - logger.info(f"Sucessfully downloaded from {url}.") + logger.warning(f"Request failed for {url}.") + else: + logger.debug(f"Successful request from {url}.") return response