diff --git a/custom_components/nordpool/__init__.py b/custom_components/nordpool/__init__.py index 61076d0..cd06441 100644 --- a/custom_components/nordpool/__init__.py +++ b/custom_components/nordpool/__init__.py @@ -32,7 +32,7 @@ NAME = DOMAIN -VERSION = "0.0.14" +VERSION = "0.0.15" ISSUEURL = "https://github.com/custom-components/nordpool/issues" STARTUP = f""" diff --git a/custom_components/nordpool/aio_price.py b/custom_components/nordpool/aio_price.py index 3c652bb..c30777c 100644 --- a/custom_components/nordpool/aio_price.py +++ b/custom_components/nordpool/aio_price.py @@ -3,17 +3,18 @@ from collections import defaultdict from datetime import date, datetime, timedelta -from homeassistant.util import dt as dt_utils -from dateutil.parser import parse as parse_dt -import backoff import aiohttp +import backoff +from dateutil.parser import parse as parse_dt +from homeassistant.util import dt as dt_utils +from nordpool.base import CurrencyMismatch from nordpool.elspot import Prices +from pytz import timezone, utc -from .misc import add_junk, exceptions_raiser +from .misc import add_junk _LOGGER = logging.getLogger(__name__) - tzs = { "DK1": "Europe/Copenhagen", "DK2": "Europe/Copenhagen", @@ -21,12 +22,11 @@ "EE": "Europe/Tallinn", "LT": "Europe/Vilnius", "LV": "Europe/Riga", - "Oslo": "Europe/Oslo", - "Kr.sand": "Europe/Oslo", - "Bergen": "Europe/Oslo", - "Molde": "Europe/Oslo", - "Tr.heim": "Europe/Oslo", - "Tromsø": "Europe/Oslo", + "NO1": "Europe/Oslo", + "NO2": "Europe/Oslo", + "NO3": "Europe/Oslo", + "NO4": "Europe/Oslo", + "NO5": "Europe/Oslo", "SE1": "Europe/Stockholm", "SE2": "Europe/Stockholm", "SE3": "Europe/Stockholm", @@ -37,10 +37,9 @@ "NL": "Europe/Amsterdam", "BE": "Europe/Brussels", "AT": "Europe/Vienna", - "DE-LU": "Europe/Berlin", + "GER": "Europe/Berlin", } - # List of page index for hourly data # Some are disabled as they don't contain the other currencies, NOK etc, # or there are some issues with data parsing for some ones' DataStartdate. @@ -140,7 +139,7 @@ async def join_result_for_correct_time(results, dt): for val in values: local = val["start"].astimezone(zone) local_end = val["end"].astimezone(zone) - if start_of_day <= local and local <= end_of_day: + if start_of_day <= local <= end_of_day: if local == local_end: _LOGGER.info( "Hour has the same start and end, most likly due to dst change %s exluded this hour", @@ -161,28 +160,120 @@ def __init__(self, currency, client, timeezone=None): super().__init__(currency) self.client = client self.timeezone = timeezone - self.API_URL_CURRENCY = "https://www.nordpoolgroup.com/api/marketdata/page/%s" + (self.HOURLY, self.DAILY, self.WEEKLY, self.MONTHLY, self.YEARLY) = ("DayAheadPrices", "AggregatePrices", + "AggregatePrices", "AggregatePrices", + "AggregatePrices") + self.API_URL = "https://dataportal-api.nordpoolgroup.com/api/%s" async def _io(self, url, **kwargs): resp = await self.client.get(url, params=kwargs) _LOGGER.debug("requested %s %s", resp.url, kwargs) + if resp.status == 204: + return None + return await resp.json() - async def _fetch_json(self, data_type, end_date=None): + def _parse_dt(self, time_str): + ''' Parse datetimes to UTC from Stockholm time, which Nord Pool uses. ''' + time = parse_dt(time_str, tzinfos={"Z": timezone("Europe/Stockholm")}) + if time.tzinfo is None: + return timezone('Europe/Stockholm').localize(time).astimezone(utc) + return time.astimezone(utc) + + def _parse_json(self, data, areas=None): + """ + Parse json response from fetcher. + Returns dictionary with + - start time + - end time + - update time + - currency + - dictionary of areas, based on selection + - list of values (dictionary with start and endtime and value) + - possible other values, such as min, max, average for hourly + """ + + if areas is None: + areas = [] + # If areas isn't a list, make it one + + if not isinstance(areas, list): + areas = list(areas) + + if data.get("status", 200) != 200 and "version" not in data: + raise Exception(f"Invalid response from Nordpool API: {data}") + + # Update currency from data + currency = data['currency'] + + # Ensure that the provided currency match the requested one + if currency != self.currency: + raise CurrencyMismatch + + start_time = None + end_time = None + + if len(data['multiAreaEntries']) > 0: + start_time = self._parse_dt(data['multiAreaEntries'][0]['deliveryStart']) + end_time = self._parse_dt(data['multiAreaEntries'][-1]['deliveryEnd']) + updated = self._parse_dt(data['updatedAt']) + + area_data = {} + + # Loop through response rows + for r in data['multiAreaEntries']: + row_start_time = self._parse_dt(r['deliveryStart']) + row_end_time = self._parse_dt(r['deliveryEnd']) + + # Loop through columns + for area_key in r['entryPerArea'].keys(): + area_price = r['entryPerArea'][area_key] + # If areas is defined and name isn't in areas, skip column + if area_key not in areas: + continue + + # If name isn't in area_data, initialize dictionary + if area_key not in area_data: + area_data[area_key] = { + 'values': [], + } + + # Append dictionary to value list + area_data[area_key]['values'].append({ + 'start': row_start_time, + 'end': row_end_time, + 'value': self._conv_to_float(area_price), + }) + + return { + 'start': start_time, + 'end': end_time, + 'updated': updated, + 'currency': currency, + 'areas': area_data + } + + async def _fetch_json(self, data_type, end_date=None, areas=None): """Fetch JSON from API""" # If end_date isn't set, default to tomorrow + if areas is None or len(areas) == 0: + raise Exception("Cannot query with empty areas") if end_date is None: end_date = date.today() + timedelta(days=1) # If end_date isn't a date or datetime object, try to parse a string if not isinstance(end_date, date) and not isinstance(end_date, datetime): end_date = parse_dt(end_date) + + return await self._io( self.API_URL % data_type, currency=self.currency, - endDate=end_date.strftime("%d-%m-%Y"), + market="DayAhead", + deliveryArea=",".join(areas), + date=end_date.strftime("%Y-%m-%d"), ) # Add more exceptions as we find them. KeyError is raised when the api return @@ -220,14 +311,14 @@ async def fetch(self, data_type, end_date=None, areas=None): tomorrow = datetime.now() + timedelta(days=1) jobs = [ - self._fetch_json(data_type, yesterday), - self._fetch_json(data_type, today), - self._fetch_json(data_type, tomorrow), + self._fetch_json(data_type, yesterday, areas), + self._fetch_json(data_type, today, areas), + self._fetch_json(data_type, tomorrow, areas), ] res = await asyncio.gather(*jobs) - raw = [await self._async_parse_json(i, areas) for i in res] - + raw = [await self._async_parse_json(i, areas) for i in res if i] + return await join_result_for_correct_time(raw, end_date) async def _async_parse_json(self, data, areas): @@ -269,6 +360,9 @@ async def yearly(self, end_date=None, areas=None): def _conv_to_float(self, s): """Convert numbers to float. Return infinity, if conversion fails.""" + # Skip if already float + if isinstance(s, float): + return s try: return float(s.replace(",", ".").replace(" ", "")) except ValueError: diff --git a/custom_components/nordpool/misc.py b/custom_components/nordpool/misc.py index e950abf..864ffe9 100644 --- a/custom_components/nordpool/misc.py +++ b/custom_components/nordpool/misc.py @@ -22,6 +22,8 @@ _LOGGER = logging.getLogger(__name__) +stockholm_tz = timezone("Europe/Stockholm") + def exceptions_raiser(): """Utility to check that all exceptions are raised.""" @@ -50,7 +52,7 @@ def add_junk(d): def stock(d): """convert datetime to stocholm time.""" - return d.astimezone(timezone("Europe/Stockholm")) + return d.astimezone(stockholm_tz) def start_of(d, typ_="hour"): diff --git a/custom_components/nordpool/sensor.py b/custom_components/nordpool/sensor.py index 38964ca..5912e84 100644 --- a/custom_components/nordpool/sensor.py +++ b/custom_components/nordpool/sensor.py @@ -38,12 +38,11 @@ "EE": ["EUR", "Estonia", 0.22], "LT": ["EUR", "Lithuania", 0.21], "LV": ["EUR", "Latvia", 0.21], - "Oslo": ["NOK", "Norway", 0.25], - "Kr.sand": ["NOK", "Norway", 0.25], - "Bergen": ["NOK", "Norway", 0.25], - "Molde": ["NOK", "Norway", 0.25], - "Tr.heim": ["NOK", "Norway", 0.25], - "Tromsø": ["NOK", "Norway", 0.25], + "NO1": ["NOK", "Norway", 0.25], + "NO2": ["NOK", "Norway", 0.25], + "NO3": ["NOK", "Norway", 0.25], + "NO4": ["NOK", "Norway", 0.25], + "NO5": ["NOK", "Norway", 0.25], "SE1": ["SEK", "Sweden", 0.25], "SE2": ["SEK", "Sweden", 0.25], "SE3": ["SEK", "Sweden", 0.25], @@ -54,9 +53,8 @@ "NL": ["EUR", "Netherlands", 0.21], "BE": ["EUR", "Belgium", 0.06], "AT": ["EUR", "Austria", 0.20], - # Tax is disabled for now, i need to split the areas - # to handle the tax. - "DE-LU": ["EUR", "Germany and Luxembourg", 0], + # Unsure about tax rate, correct if wrong + "GER": ["EUR", "Germany", 0.23], } # Needed incase a user wants the prices in non local currency @@ -64,7 +62,7 @@ _CURRENTY_TO_CENTS = {"DKK": "Øre", "NOK": "Øre", "SEK": "Öre", "EUR": "c"} DEFAULT_CURRENCY = "NOK" -DEFAULT_REGION = "Kr.sand" +DEFAULT_REGION = list(_REGIONS.keys())[0] DEFAULT_NAME = "Elspot" diff --git a/custom_components/nordpool/test_parser.py b/custom_components/nordpool/test_parser.py index f9a1282..b344057 100644 --- a/custom_components/nordpool/test_parser.py +++ b/custom_components/nordpool/test_parser.py @@ -1,42 +1,232 @@ +import asyncio import logging -import sys # Make sure plexapi is in the systempath from collections import defaultdict from datetime import date, datetime, timedelta from operator import itemgetter -from os.path import abspath, dirname -from pprint import pprint -import requests +import aiohttp +from aiozoneinfo import async_get_time_zone # https://repl.it/repls/WildImpishMass from dateutil import tz from dateutil.parser import parse as parse_dt -from nordpool import elspot +from nordpool.base import CurrencyMismatch +from nordpool.elspot import Prices +_LOGGER = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) + + +INVALID_VALUES = frozenset((None, float("inf"))) + + +class InvalidValueException(ValueError): + pass + + + +class AioPrices(Prices): + """Interface""" -class PP(elspot.Prices): - def __init__(self, currency): + def __init__(self, currency, client, timeezone=None): super().__init__(currency) - self.API_URL_CURRENCY = "https://www.nordpoolgroup.com/api/marketdata/page/%s" + self.client = client + self.timeezone = timeezone + (self.HOURLY, self.DAILY, self.WEEKLY, self.MONTHLY, self.YEARLY) = ("DayAheadPrices", "AggregatePrices", + "AggregatePrices", "AggregatePrices", + "AggregatePrices") + self.API_URL = "https://dataportal-api.nordpoolgroup.com/api/%s" - def _fetch_json(self, data_type, end_date=None): - ''' Fetch JSON from API ''' + async def _io(self, url, **kwargs): + + resp = await self.client.get(url, params=kwargs, headers={ + 'Origin': 'https://data.nordpoolgroup.com' + }) + _LOGGER.debug("requested %s %s", resp.url, kwargs) + + return await resp.json() + + def _parse_json(self, data, areas=None): + """ + Parse json response from fetcher. + Returns dictionary with + - start time + - end time + - update time + - currency + - dictionary of areas, based on selection + - list of values (dictionary with start and endtime and value) + - possible other values, such as min, max, average for hourly + """ + + # If areas isn't a list, make it one + if areas is None: + areas = [] + if not isinstance(areas, list): + areas = list(areas) + + if data.get("status", 200) != 200 and "version" not in data: + raise Exception(f"Invalid response from Nordpool API: {data}") + + # Update currency from data + currency = data['currency'] + + # Ensure that the provided currency match the requested one + if currency != self.currency: + raise CurrencyMismatch + + start_time = None + end_time = None + + if len(data['multiAreaEntries']) > 0: + start_time = self._parse_dt(data['multiAreaEntries'][0]['deliveryStart']) + end_time = self._parse_dt(data['multiAreaEntries'][-1]['deliveryEnd']) + updated = self._parse_dt(data['updatedAt']) + + area_data = {} + + # Loop through response rows + for r in data['multiAreaEntries']: + row_start_time = self._parse_dt(r['deliveryStart']) + row_end_time = self._parse_dt(r['deliveryEnd']) + + # Loop through columns + for area_key in r['entryPerArea'].keys(): + area_price = r['entryPerArea'][area_key] + # If areas is defined and name isn't in areas, skip column + if area_key not in areas: + continue + + # If name isn't in area_data, initialize dictionary + if area_key not in area_data: + area_data[area_key] = { + 'values': [], + } + + # Append dictionary to value list + area_data[area_key]['values'].append({ + 'start': row_start_time, + 'end': row_end_time, + 'value': self._conv_to_float(area_price), + }) + + return { + 'start': start_time, + 'end': end_time, + 'updated': updated, + 'currency': currency, + 'areas': area_data + } + + async def _fetch_json(self, data_type, end_date=None, areas=None): + """Fetch JSON from API""" # If end_date isn't set, default to tomorrow + if areas is None: + areas = [] if end_date is None: end_date = date.today() + timedelta(days=1) # If end_date isn't a date or datetime object, try to parse a string if not isinstance(end_date, date) and not isinstance(end_date, datetime): end_date = parse_dt(end_date) - if self.currency != "EUR": - data_type = 23 - # Create request to API - r = requests.get(self.API_URL % data_type, params={ - 'currency': self.currency, - 'endDate': end_date.strftime('%d-%m-%Y'), - }) - # Return JSON response - return r.json() + + return await self._io( + self.API_URL % data_type, + currency=self.currency, + market="DayAhead", + deliveryArea=",".join(areas), + date=end_date.strftime("%Y-%m-%d"), + ) + + # Add more exceptions as we find them. KeyError is raised when the api return + # junk due to currency not being available in the data. + + async def fetch(self, data_type, end_date=None, areas=None): + """ + Fetch data from API. + Inputs: + - data_type + API page id, one of Prices.HOURLY, Prices.DAILY etc + - end_date + datetime to end the data fetching + defaults to tomorrow + - areas + list of areas to fetch, such as ['SE1', 'SE2', 'FI'] + defaults to all areas + Returns dictionary with + - start time + - end time + - update time + - currency + - dictionary of areas, based on selection + - list of values (dictionary with start and endtime and value) + - possible other values, such as min, max, average for hourly + """ + if areas is None: + areas = [] + + yesterday = datetime.now() - timedelta(days=1) + today = datetime.now() + tomorrow = datetime.now() + timedelta(days=1) + + jobs = [ + self._fetch_json(data_type, yesterday, areas), + self._fetch_json(data_type, today, areas), + self._fetch_json(data_type, tomorrow, areas), + ] + + res = await asyncio.gather(*jobs) + raw = [await self._async_parse_json(i, areas) for i in res] + + return await join_result_for_correct_time(raw, end_date) + + async def _async_parse_json(self, data, areas): + """ + Async version of _parse_json to prevent blocking calls inside the event loop. + """ + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, self._parse_json, data, areas) + + async def hourly(self, end_date=None, areas=None): + """Helper to fetch hourly data, see Prices.fetch()""" + if areas is None: + areas = [] + return await self.fetch(self.HOURLY, end_date, areas) + + async def daily(self, end_date=None, areas=None): + """Helper to fetch daily data, see Prices.fetch()""" + if areas is None: + areas = [] + return await self.fetch(self.DAILY, end_date, areas) + + async def weekly(self, end_date=None, areas=None): + """Helper to fetch weekly data, see Prices.fetch()""" + if areas is None: + areas = [] + return await self.fetch(self.WEEKLY, end_date, areas) + + async def monthly(self, end_date=None, areas=None): + """Helper to fetch monthly data, see Prices.fetch()""" + if areas is None: + areas = [] + return await self.fetch(self.MONTHLY, end_date, areas) + + async def yearly(self, end_date=None, areas=None): + """Helper to fe +tch yearly data, see Prices.fetch()""" + if areas is None: + areas = [] + return await self.fetch(self.YEARLY, end_date, areas) + + def _conv_to_float(self, s): + """Convert numbers to float. Return infinity, if conversion fails.""" + # Skip if already float + if isinstance(s, float): + return s + try: + return float(s.replace(",", ".").replace(" ", "")) + except ValueError: + return float("inf") @@ -51,8 +241,6 @@ def _fetch_json(self, data_type, end_date=None): -_LOGGER = logging.getLogger(__name__) -logging.basicConfig(level=logging.DEBUG) tzs = { @@ -89,26 +277,23 @@ def add_junk(d): return d -def join_result_for_correct_time(results): +async def join_result_for_correct_time(results, dt): """Parse a list of responses from the api - to extract the correct hours in there timezone. - - + to extract the correct hours in there timezone. """ - utc = datetime.utcnow() + # utc = datetime.utcnow() fin = defaultdict(dict) + # _LOGGER.debug("join_result_for_correct_time %s", dt) + utc = dt - n = 0 for day_ in results: - n += 1 - print(n) for key, value in day_.get("areas", {}).items(): zone = tzs.get(key) if zone is None: _LOGGER.debug("Skipping %s", key) continue else: - zone = tz.gettz(zone) + zone = await async_get_time_zone(zone) # We add junk here as the peak etc # from the api is based on cet, not the @@ -136,12 +321,15 @@ def join_result_for_correct_time(results): local = val["start"].astimezone(zone) local_end = val["end"].astimezone(zone) if start_of_day <= local and local <= end_of_day: - - # - if n == 1: - # this is yesterday - print("outlier %s %s %s %s" % (key, val["start"], val["end"], val["value"])) - fin["areas"][key]["values"].append(val) + if local == local_end: + _LOGGER.info( + "Hour has the same start and end, most likly due to dst change %s exluded this hour", + val, + ) + elif val['value'] in INVALID_VALUES: + raise InvalidValueException(f"Invalid value in {val} for area '{key}'") + else: + fin["areas"][key]["values"].append(val) return fin @@ -149,13 +337,13 @@ def join_result_for_correct_time(results): if __name__ == "__main__": - import click + import asyncclick as click @click.command() @click.option('--region', '-r', default="Kr.sand") @click.option('--currency', '-c', default="NOK") @click.option('--vat', '-v', default=0) - def manual_check(region, currency, vat): + async def manual_check(region, currency, vat): ts = tz.gettz(tzs[region]) @@ -163,26 +351,26 @@ def manual_check(region, currency, vat): # Convert time zone lt = utc.astimezone(ts) - dt_today = lt.date() + dt_today = lt dt_yesterday = lt + timedelta(days=-1) - spot = PP(currency) - yesterday = spot.hourly(end_date=dt_yesterday) - today = spot.hourly(end_date=dt_today) - tomorrow = spot.hourly(end_date=dt_today + timedelta(days=1)) + spot = AioPrices(currency, aiohttp.client.ClientSession()) + yesterday = await spot.hourly(end_date=dt_yesterday, areas=[region]) + today = await spot.hourly(end_date=dt_today, areas=[region]) + tomorrow = await spot.hourly(end_date=dt_today + timedelta(days=1), areas=[region]) #print(today) #print(pprint(today.get("areas"))) #return results = [yesterday, today, tomorrow] - data = join_result_for_correct_time(results) + rsults = await join_result_for_correct_time(results, dt_today) + values = [] - return - for key, value in data["areas"].items(): + for key, value in rsults["areas"].items(): values = [] if key == region or region is None: - for v in data["areas"][key]["values"]: + for v in rsults["areas"][key]["values"]: zone = tzs.get(key) if zone is None: continue @@ -202,4 +390,4 @@ def manual_check(region, currency, vat): if len(values): print("total hours %s" % len(values)) - manual_check() + asyncio.run(manual_check())