Skip to content

Commit

Permalink
Updates parser to use new proxy
Browse files Browse the repository at this point in the history
  • Loading branch information
madsnedergaard committed Jan 5, 2024
1 parent a46aa85 commit 5af90b0
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 37 deletions.
55 changes: 19 additions & 36 deletions parsers/ENTSOE.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,7 @@

SOURCE = "entsoe.eu"

ENDPOINT = "/api"
ENTSOE_HOST = "https://web-api.tp.entsoe.eu"


EU_PROXY = "https://eu-proxy-jfnx5klx2a-ew.a.run.app{endpoint}?host={host}"

ENTSOE_ENDPOINT = ENTSOE_HOST + ENDPOINT
ENTSOE_EU_PROXY_ENDPOINT = EU_PROXY.format(endpoint=ENDPOINT, host=ENTSOE_HOST)
ENTSOE_URL = "https://entsoe-proxy-jfnx5klx2a-ew.a.run.app"

ENTSOE_PARAMETER_DESC = {
"B01": "Biomass",
Expand Down Expand Up @@ -487,12 +480,8 @@ def query_ENTSOE(
Raises an exception if no API token is found.
Returns a request object.
"""
env_var = "ENTSOE_REFETCH_TOKEN"
url = ENTSOE_EU_PROXY_ENDPOINT
if target_datetime is None:
target_datetime = datetime.now(timezone.utc)
env_var = "ENTSOE_TOKEN"
url = ENTSOE_ENDPOINT

if not isinstance(target_datetime, datetime):
raise ParserException(
Expand All @@ -508,31 +497,25 @@ def query_ENTSOE(
"%Y%m%d%H00" # YYYYMMDDHH00
)

# Due to rate limiting, we need to spread our requests across different tokens
tokens = get_token(env_var).split(",")
# Shuffle the tokens so that we don't always use the same one first.
shuffle(tokens)
last_response_if_all_fail = None
# Try each token until we get a valid response
for token in tokens:
params["securityToken"] = token
response: Response = session.get(url, params=params)
if response.ok:
return response.text
else:
last_response_if_all_fail = response
# If we get here, all tokens failed to fetch valid data
# and we will check the last response for a error message.
token = get_token("ENTSOE_TOKEN")
params["securityToken"] = token
response: Response = session.get(ENTSOE_URL, params=params)
if response.ok:
return response.text

# If we get here, the request failed to fetch valid data
# and we will check the response for an error message
exception_message = None
if last_response_if_all_fail is not None:
soup = BeautifulSoup(last_response_if_all_fail.text, "html.parser")
text = soup.find_all("text")
if len(text):
error_text = soup.find_all("text")[0].prettify()
if "No matching data found" in error_text:
exception_message = "No matching data found"
if exception_message is None:
exception_message = f"Status code: [{last_response_if_all_fail.status_code}]. Reason: {last_response_if_all_fail.reason}"
soup = BeautifulSoup(response.text, "html.parser")
text = soup.find_all("text")
if len(text):
error_text = soup.find_all("text")[0].prettify()
if "No matching data found" in error_text:
exception_message = "No matching data found"
if exception_message is None:
exception_message = (
f"Status code: [{response.status_code}]. Reason: {response.reason}"
)

raise ParserException(
parser="ENTSOE.py",
Expand Down
2 changes: 1 addition & 1 deletion scripts/ENTSOE_capacity_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ def main():
sys.exit(1)
data = parse_from_csv(data_file)
else:
token = args.api_token or get_token("ENTSOE_TOKEN").split(",")[0]
token = args.api_token or get_token("ENTSOE_TOKEN")
if token is None:
print(
"ERROR: If no CSV file is given, the option --api-token must be provided",
Expand Down

0 comments on commit 5af90b0

Please sign in to comment.