From 5af90b07528f1deb6738cf14cc32be2802222a3e Mon Sep 17 00:00:00 2001 From: madsnedergaard Date: Fri, 5 Jan 2024 10:43:29 +0100 Subject: [PATCH 1/3] Updates parser to use new proxy --- parsers/ENTSOE.py | 55 +++++++++++-------------------- scripts/ENTSOE_capacity_update.py | 2 +- 2 files changed, 20 insertions(+), 37 deletions(-) diff --git a/parsers/ENTSOE.py b/parsers/ENTSOE.py index 10dfcd52ee..564af332b0 100644 --- a/parsers/ENTSOE.py +++ b/parsers/ENTSOE.py @@ -41,14 +41,7 @@ SOURCE = "entsoe.eu" -ENDPOINT = "/api" -ENTSOE_HOST = "https://web-api.tp.entsoe.eu" - - -EU_PROXY = "https://eu-proxy-jfnx5klx2a-ew.a.run.app{endpoint}?host={host}" - -ENTSOE_ENDPOINT = ENTSOE_HOST + ENDPOINT -ENTSOE_EU_PROXY_ENDPOINT = EU_PROXY.format(endpoint=ENDPOINT, host=ENTSOE_HOST) +ENTSOE_URL = "https://entsoe-proxy-jfnx5klx2a-ew.a.run.app" ENTSOE_PARAMETER_DESC = { "B01": "Biomass", @@ -487,12 +480,8 @@ def query_ENTSOE( Raises an exception if no API token is found. Returns a request object. """ - env_var = "ENTSOE_REFETCH_TOKEN" - url = ENTSOE_EU_PROXY_ENDPOINT if target_datetime is None: target_datetime = datetime.now(timezone.utc) - env_var = "ENTSOE_TOKEN" - url = ENTSOE_ENDPOINT if not isinstance(target_datetime, datetime): raise ParserException( @@ -508,31 +497,25 @@ def query_ENTSOE( "%Y%m%d%H00" # YYYYMMDDHH00 ) - # Due to rate limiting, we need to spread our requests across different tokens - tokens = get_token(env_var).split(",") - # Shuffle the tokens so that we don't always use the same one first. - shuffle(tokens) - last_response_if_all_fail = None - # Try each token until we get a valid response - for token in tokens: - params["securityToken"] = token - response: Response = session.get(url, params=params) - if response.ok: - return response.text - else: - last_response_if_all_fail = response - # If we get here, all tokens failed to fetch valid data - # and we will check the last response for a error message. + token = get_token("ENTSOE_TOKEN") + params["securityToken"] = token + response: Response = session.get(ENTSOE_URL, params=params) + if response.ok: + return response.text + + # If we get here, the request failed to fetch valid data + # and we will check the response for an error message exception_message = None - if last_response_if_all_fail is not None: - soup = BeautifulSoup(last_response_if_all_fail.text, "html.parser") - text = soup.find_all("text") - if len(text): - error_text = soup.find_all("text")[0].prettify() - if "No matching data found" in error_text: - exception_message = "No matching data found" - if exception_message is None: - exception_message = f"Status code: [{last_response_if_all_fail.status_code}]. Reason: {last_response_if_all_fail.reason}" + soup = BeautifulSoup(response.text, "html.parser") + text = soup.find_all("text") + if len(text): + error_text = soup.find_all("text")[0].prettify() + if "No matching data found" in error_text: + exception_message = "No matching data found" + if exception_message is None: + exception_message = ( + f"Status code: [{response.status_code}]. Reason: {response.reason}" + ) raise ParserException( parser="ENTSOE.py", diff --git a/scripts/ENTSOE_capacity_update.py b/scripts/ENTSOE_capacity_update.py index 8113fb49fb..7c0d636052 100644 --- a/scripts/ENTSOE_capacity_update.py +++ b/scripts/ENTSOE_capacity_update.py @@ -148,7 +148,7 @@ def main(): sys.exit(1) data = parse_from_csv(data_file) else: - token = args.api_token or get_token("ENTSOE_TOKEN").split(",")[0] + token = args.api_token or get_token("ENTSOE_TOKEN") if token is None: print( "ERROR: If no CSV file is given, the option --api-token must be provided", From eb004330d10b712f0a379a5be288c3ee7814c8d8 Mon Sep 17 00:00:00 2001 From: madsnedergaard Date: Fri, 5 Jan 2024 11:33:30 +0100 Subject: [PATCH 2/3] Runs formatting --- parsers/ENTSOE.py | 1 - 1 file changed, 1 deletion(-) diff --git a/parsers/ENTSOE.py b/parsers/ENTSOE.py index 564af332b0..2095aaef4e 100644 --- a/parsers/ENTSOE.py +++ b/parsers/ENTSOE.py @@ -15,7 +15,6 @@ import re from datetime import datetime, timedelta, timezone from logging import Logger, getLogger -from random import shuffle from typing import Any import arrow From 4a7f7869bbbd33e6d89dbe77720143936316fec0 Mon Sep 17 00:00:00 2001 From: madsnedergaard Date: Fri, 5 Jan 2024 11:39:35 +0100 Subject: [PATCH 3/3] Removes tests that are no longer relevant --- parsers/test/test_ENTSOE.py | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/parsers/test/test_ENTSOE.py b/parsers/test/test_ENTSOE.py index 1545d555d2..475a8c163b 100644 --- a/parsers/test/test_ENTSOE.py +++ b/parsers/test/test_ENTSOE.py @@ -162,39 +162,6 @@ def test_fetch_with_negative_values(self): class TestENTSOE_Refetch(unittest.TestCase): - def test_refetch_token(self) -> None: - token = mock.Mock(return_value="token") - with mock.patch("parsers.ENTSOE.get_token", token) as patched_get_token: - self.session = Session() - self.adapter = Adapter() - self.session.mount("https://", self.adapter) - with open("parsers/test/mocks/ENTSOE/FR_prices.xml", "rb") as price_fr_data: - self.adapter.register_uri( - GET, - ANY, - content=price_fr_data.read(), - ) - _ = ENTSOE.fetch_price(ZoneKey("DE"), self.session) - patched_get_token.assert_called_once_with("ENTSOE_TOKEN") - patched_get_token.reset_mock() - _ = ENTSOE.fetch_price( - ZoneKey("DE"), self.session, datetime(2021, 1, 1) - ) - patched_get_token.assert_called_once_with("ENTSOE_REFETCH_TOKEN") - - def test_refetch_uses_proxy(self): - os.environ["ENTSOE_REFETCH_TOKEN"] = "proxy" - self.session = Session() - self.adapter = Adapter() - self.session.mount("https://", self.adapter) - with open("parsers/test/mocks/ENTSOE/FR_prices.xml", "rb") as price_fr_data: - self.adapter.register_uri( - GET, - ENTSOE.ENTSOE_EU_PROXY_ENDPOINT, - content=price_fr_data.read(), - ) - _ = ENTSOE.fetch_price(ZoneKey("DE"), self.session, datetime(2021, 1, 1)) - def test_fetch_uses_normal_url(self): os.environ["ENTSOE_TOKEN"] = "proxy" self.session = Session() @@ -203,7 +170,7 @@ def test_fetch_uses_normal_url(self): with open("parsers/test/mocks/ENTSOE/FR_prices.xml", "rb") as price_fr_data: self.adapter.register_uri( GET, - ENTSOE.ENTSOE_ENDPOINT, + ENTSOE.ENTSOE_URL, content=price_fr_data.read(), ) _ = ENTSOE.fetch_price(ZoneKey("DE"), self.session)