From 76939a547c655857678cc2b5a16ace65c9c9497c Mon Sep 17 00:00:00 2001 From: Mathilde Daugy Date: Thu, 15 Dec 2022 18:29:34 +0100 Subject: [PATCH 01/15] updated parser --- parsers/US_PJM.py | 105 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 21 deletions(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 4708d769cd..3ff23b1b88 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -4,17 +4,21 @@ import json import re -from datetime import datetime +from datetime import datetime, timedelta from logging import Logger, getLogger -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union import arrow import demjson3 as demjson +import pandas as pd +import pytz from bs4 import BeautifulSoup from dateutil import parser, tz from requests import Session, get -from .lib.utils import get_token +from parsers.lib.config import refetch_frequency +from parsers.lib.exceptions import ParserException +from parsers.lib.utils import get_token # Used for consumption forecast data. API_ENDPOINT = "https://api.pjm.com/api/v1/" @@ -48,6 +52,20 @@ "cpl east": "SOUTHIMP|CPLE", } +FUEL_MAPPING = { + "Coal": "coal", + "Gas": "gas", + "Hydro": "hydro", + "Multiple Fuels": "unknown", + "Nuclear": "nuclear", + "Oil": "oil", + "Other": "unknown", + "Other Renewables": "unknown", + "Solar": "solar", + "Storage": "battery", + "Wind": "wind", +} + def extract_data(session: Optional[Session] = None) -> tuple: """ @@ -125,6 +143,26 @@ def data_processer(data) -> dict: return production +def fetch_api_data(kind: str, params: dict) -> List[Dict]: + + headers = { + "Host": "api.pjm.com", + "Ocp-Apim-Subscription-Key": get_token("PJM_TOKEN"), + "Origin": "http://dataminer2.pjm.com", + "Referer": "http://dataminer2.pjm.com/", + } + url = API_ENDPOINT + kind + resp = get(url, params, headers=headers) + if resp.status_code == 200: + data = resp.json() + return data + else: + raise ParserException( + parser="US_PJM.py", + message=f"{kind} data is not available in the API", + ) + + def fetch_consumption_forecast_7_days( zone_key: str = "US-PJM", session: Optional[Session] = None, @@ -145,9 +183,7 @@ def fetch_consumption_forecast_7_days( params = {"download": True, "startRow": 1, "forecast_area": "RTO_COMBINED"} # query API - url = API_ENDPOINT + "load_frcstd_7_day" - resp = get(url, params, headers=headers) - data = json.loads(resp.content) + data = fetch_api_data(kind="load_frcstd_7_day", params=params) data_points = list() for elem in data: @@ -163,28 +199,55 @@ def fetch_consumption_forecast_7_days( return data_points +@refetch_frequency(timedelta(days=1)) def fetch_production( zone_key: str = "US-PJM", session: Optional[Session] = None, target_datetime: Optional[datetime] = None, logger: Logger = getLogger(__name__), -) -> dict: - """Requests the last known production mix (in MW) of a given country.""" - if target_datetime is not None: - raise NotImplementedError("This parser is not yet able to parse past dates") - - extracted = extract_data(session=None) - production = data_processer(extracted[0]) +) -> List[Dict]: + """uses PJM API to get generation by fuel. we assume that storage is battery storage (see https://learn.pjm.com/energy-innovations/energy-storage)""" + if target_datetime is None: + target_datetime = arrow.utcnow().datetime + if not session: + session = Session() - datapoint = { - "zoneKey": zone_key, - "datetime": extracted[1], - "production": production, - "storage": {"hydro": None, "battery": None}, - "source": "pjm.com", + params = { + "download": True, + "startRow": 1, + "fields": "datetime_beginning_ept,fuel_type,mw", + "datetime_beginning_ept": target_datetime.strftime("%Y-%m-%dT%H:00:00.0000000"), } - - return datapoint + resp_data = fetch_api_data(kind="gen_by_fuel", params=params) + + data = pd.DataFrame(resp_data) + data.fuel_type = data.fuel_type.map(FUEL_MAPPING) + + all_data_points = [] + for dt in set(data.datetime_beginning_ept): + production = {} + storage = {} + data_dt = data.loc[data.datetime_beginning_ept == dt] + for mode in set(data.fuel_type): + if mode == "battery": + storage["battery"] = data_dt.loc[ + data_dt.fuel_type == mode, "mw" + ].values[0] + else: + production[mode] = data_dt.loc[data_dt.fuel_type == mode, "mw"].values[ + 0 + ] + data_point = { + "zoneKey": zone_key, + "datetime": arrow.get(dt).datetime.replace( + tzinfo=pytz.timezone("US/Eastern") + ), + "production": production, + "storage": storage, + "source": "pjm.com", + } + all_data_points += [data_point] + return all_data_points def add_default_tz(timestamp): From 21e9c978641ee38fcf6526daca50cc84af7d104a Mon Sep 17 00:00:00 2001 From: Mathilde Daugy Date: Thu, 15 Dec 2022 18:35:15 +0100 Subject: [PATCH 02/15] clean up --- parsers/US_PJM.py | 75 ----------------------------------------------- 1 file changed, 75 deletions(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 3ff23b1b88..fcc54b4e81 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -67,81 +67,6 @@ } -def extract_data(session: Optional[Session] = None) -> tuple: - """ - Makes a request to the PJM data url. - Finds timestamp of current data and converts into a useful form. - Finds generation data inside script tag. - - :return: tuple of generation data and datetime. - """ - - s = session or Session() - req = s.get(url) - soup = BeautifulSoup(req.content, "html.parser") - - try: - time_div = soup.find("div", id="asOfDate").text - except AttributeError: - raise LookupError("No data is available for US-PJM.") - - time_pattern = re.compile( - r"""(\d{1,2} #Hour can be 1/2 digits. - : #Separator. - \d{2})\s #Minutes must be 2 digits with a space after. - (a.m.|p.m.) #Either am or pm allowed.""", - re.X, - ) - - latest_time = re.search(time_pattern, time_div) - - time_data = latest_time.group(1).split(":") - am_or_pm = latest_time.group(2) - hour = int(time_data[0]) - minute = int(time_data[1]) - - # Time format used by PJM is slightly unusual and needs to be converted so arrow can use it. - if am_or_pm == "p.m." and hour != 12: - # Time needs to be in 24hr format - hour += 12 - elif am_or_pm == "a.m." and hour == 12: - # Midnight is 12 a.m. - hour = 0 - - arr_dt = arrow.now("America/New_York").replace(hour=hour, minute=minute) - future_check = arrow.now("America/New_York") - - if arr_dt > future_check: - # Generation mix lags 1-2hrs behind present. - # This check prevents data near midnight being given the wrong date. - arr_dt = arr_dt.shift(days=-1) - - dt = arr_dt.floor("minute").datetime - - generation_mix_div = soup.find("div", id="rtschartallfuelspjmGenFuelM_container") - generation_mix_script = generation_mix_div.next_sibling - - pattern = r"series: \[(.*)\]" - script_data = re.search(pattern, str(generation_mix_script)).group(1) - - # demjson is required because script data is javascript not valid json. - raw_data = demjson.decode(script_data) - data = raw_data["data"] - - return data, dt - - -def data_processer(data) -> dict: - """Takes a list of dictionaries and extracts generation type and value from each.""" - - production = {} - for point in data: - gen_type = mapping[point["name"]] - gen_value = float(point["y"]) - production[gen_type] = production.get(gen_type, 0.0) + gen_value - - return production - def fetch_api_data(kind: str, params: dict) -> List[Dict]: From 864065418a3ed0e34264e8642b611fe77da08f76 Mon Sep 17 00:00:00 2001 From: Mathilde Daugy Date: Thu, 15 Dec 2022 18:43:01 +0100 Subject: [PATCH 03/15] format --- parsers/US_PJM.py | 1 - 1 file changed, 1 deletion(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index fcc54b4e81..223e6d2c5a 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -67,7 +67,6 @@ } - def fetch_api_data(kind: str, params: dict) -> List[Dict]: headers = { From 98848bc42e8aa338bf1c5b9908ad1dafa071673f Mon Sep 17 00:00:00 2001 From: Mathilde Daugy Date: Thu, 15 Dec 2022 19:07:02 +0100 Subject: [PATCH 04/15] fixed test error --- parsers/US_PJM.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 223e6d2c5a..961f9592a7 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -67,7 +67,7 @@ } -def fetch_api_data(kind: str, params: dict) -> List[Dict]: +def fetch_api_data(kind: str, params: dict) -> list: headers = { "Host": "api.pjm.com", @@ -129,7 +129,7 @@ def fetch_production( session: Optional[Session] = None, target_datetime: Optional[datetime] = None, logger: Logger = getLogger(__name__), -) -> List[Dict]: +) -> list: """uses PJM API to get generation by fuel. we assume that storage is battery storage (see https://learn.pjm.com/energy-innovations/energy-storage)""" if target_datetime is None: target_datetime = arrow.utcnow().datetime From 37aa3a707e66822d54f0bad59b83a11fdba47d9d Mon Sep 17 00:00:00 2001 From: Mathilde Daugy <107848894+mathilde-daugy@users.noreply.github.com> Date: Fri, 16 Dec 2022 09:14:18 +0100 Subject: [PATCH 05/15] Update parsers/US_PJM.py Co-authored-by: Viktor Andersson <30777521+VIKTORVAV99@users.noreply.github.com> --- parsers/US_PJM.py | 1 - 1 file changed, 1 deletion(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 961f9592a7..7028aca2be 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -2,7 +2,6 @@ """Parser for the PJM area of the United States.""" -import json import re from datetime import datetime, timedelta from logging import Logger, getLogger From ad83ec2092d9276efb1cfbf445203702ef0fc61a Mon Sep 17 00:00:00 2001 From: Mathilde Daugy <107848894+mathilde-daugy@users.noreply.github.com> Date: Fri, 16 Dec 2022 09:14:23 +0100 Subject: [PATCH 06/15] Update parsers/US_PJM.py Co-authored-by: Viktor Andersson <30777521+VIKTORVAV99@users.noreply.github.com> --- parsers/US_PJM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 7028aca2be..769d6a9bb5 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -5,7 +5,7 @@ import re from datetime import datetime, timedelta from logging import Logger, getLogger -from typing import Dict, List, Optional, Union +from typing import List, Optional, Union import arrow import demjson3 as demjson From 6fc032ace2b6fc03d7141f195ecd4a687b323c51 Mon Sep 17 00:00:00 2001 From: Mathilde Daugy <107848894+mathilde-daugy@users.noreply.github.com> Date: Fri, 16 Dec 2022 09:14:29 +0100 Subject: [PATCH 07/15] Update parsers/US_PJM.py Co-authored-by: Viktor Andersson <30777521+VIKTORVAV99@users.noreply.github.com> --- parsers/US_PJM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 769d6a9bb5..12f1c3b1a6 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -75,7 +75,7 @@ def fetch_api_data(kind: str, params: dict) -> list: "Referer": "http://dataminer2.pjm.com/", } url = API_ENDPOINT + kind - resp = get(url, params, headers=headers) + resp: Response = session.get(url, params, headers=headers) if resp.status_code == 200: data = resp.json() return data From 21833c9eec10e954ca36d60803671b3f656b0d6a Mon Sep 17 00:00:00 2001 From: Mathilde Daugy <107848894+mathilde-daugy@users.noreply.github.com> Date: Fri, 16 Dec 2022 09:14:38 +0100 Subject: [PATCH 08/15] Update parsers/US_PJM.py Co-authored-by: Viktor Andersson <30777521+VIKTORVAV99@users.noreply.github.com> --- parsers/US_PJM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 12f1c3b1a6..cb70b706a6 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -13,7 +13,7 @@ import pytz from bs4 import BeautifulSoup from dateutil import parser, tz -from requests import Session, get +from requests import Session, Response from parsers.lib.config import refetch_frequency from parsers.lib.exceptions import ParserException From 53265aa36ae1663add3238a90ca1d5753d56e93a Mon Sep 17 00:00:00 2001 From: Mathilde Daugy <107848894+mathilde-daugy@users.noreply.github.com> Date: Fri, 16 Dec 2022 09:15:01 +0100 Subject: [PATCH 09/15] Update parsers/US_PJM.py Co-authored-by: Viktor Andersson <30777521+VIKTORVAV99@users.noreply.github.com> --- parsers/US_PJM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index cb70b706a6..b855d968a8 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -66,7 +66,7 @@ } -def fetch_api_data(kind: str, params: dict) -> list: +def fetch_api_data(kind: str, params: dict, session: Session) -> list: headers = { "Host": "api.pjm.com", From 625630c1d5297cfd0f70bb07e71dd4e70e583cd8 Mon Sep 17 00:00:00 2001 From: Mathilde Daugy <107848894+mathilde-daugy@users.noreply.github.com> Date: Fri, 16 Dec 2022 09:15:17 +0100 Subject: [PATCH 10/15] Update parsers/US_PJM.py Co-authored-by: Viktor Andersson <30777521+VIKTORVAV99@users.noreply.github.com> --- parsers/US_PJM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index b855d968a8..48d6bb91f3 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -106,7 +106,7 @@ def fetch_consumption_forecast_7_days( params = {"download": True, "startRow": 1, "forecast_area": "RTO_COMBINED"} # query API - data = fetch_api_data(kind="load_frcstd_7_day", params=params) + data = fetch_api_data(kind="load_frcstd_7_day", params=params, session=session) data_points = list() for elem in data: From eedf763cf3c2981a7d3a5d714baa84112b4e0df9 Mon Sep 17 00:00:00 2001 From: Mathilde Daugy <107848894+mathilde-daugy@users.noreply.github.com> Date: Fri, 16 Dec 2022 09:15:32 +0100 Subject: [PATCH 11/15] Update parsers/US_PJM.py Co-authored-by: Viktor Andersson <30777521+VIKTORVAV99@users.noreply.github.com> --- parsers/US_PJM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 48d6bb91f3..d6246e0b1c 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -125,7 +125,7 @@ def fetch_consumption_forecast_7_days( @refetch_frequency(timedelta(days=1)) def fetch_production( zone_key: str = "US-PJM", - session: Optional[Session] = None, + session: Session = Session(), target_datetime: Optional[datetime] = None, logger: Logger = getLogger(__name__), ) -> list: From 65be30257e3298111bc99856cbfe7c60b1c37527 Mon Sep 17 00:00:00 2001 From: Mathilde Daugy <107848894+mathilde-daugy@users.noreply.github.com> Date: Fri, 16 Dec 2022 09:16:16 +0100 Subject: [PATCH 12/15] Update parsers/US_PJM.py Co-authored-by: Viktor Andersson <30777521+VIKTORVAV99@users.noreply.github.com> --- parsers/US_PJM.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index d6246e0b1c..5e15e8cf12 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -132,8 +132,6 @@ def fetch_production( """uses PJM API to get generation by fuel. we assume that storage is battery storage (see https://learn.pjm.com/energy-innovations/energy-storage)""" if target_datetime is None: target_datetime = arrow.utcnow().datetime - if not session: - session = Session() params = { "download": True, From 2aed39fd6de54bcf070f8abab353bedc8948aac8 Mon Sep 17 00:00:00 2001 From: Viktor Andersson <30777521+VIKTORVAV99@users.noreply.github.com> Date: Fri, 16 Dec 2022 17:30:55 +0100 Subject: [PATCH 13/15] Session cleanup --- parsers/US_PJM.py | 51 +++++++++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 5e15e8cf12..b05d68638b 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -13,7 +13,7 @@ import pytz from bs4 import BeautifulSoup from dateutil import parser, tz -from requests import Session, Response +from requests import Response, Session from parsers.lib.config import refetch_frequency from parsers.lib.exceptions import ParserException @@ -75,7 +75,7 @@ def fetch_api_data(kind: str, params: dict, session: Session) -> list: "Referer": "http://dataminer2.pjm.com/", } url = API_ENDPOINT + kind - resp: Response = session.get(url, params, headers=headers) + resp: Response = session.get(url, params=params, headers=headers) if resp.status_code == 200: data = resp.json() return data @@ -88,7 +88,7 @@ def fetch_api_data(kind: str, params: dict, session: Session) -> list: def fetch_consumption_forecast_7_days( zone_key: str = "US-PJM", - session: Optional[Session] = None, + session: Session = Session(), target_datetime: Optional[datetime] = None, logger: Logger = getLogger(__name__), ) -> list: @@ -96,8 +96,6 @@ def fetch_consumption_forecast_7_days( if target_datetime: raise NotImplementedError("This parser is not yet able to parse past dates") - if not session: - session = Session() headers = {"Ocp-Apim-Subscription-Key": get_token("PJM_TOKEN")} @@ -139,7 +137,7 @@ def fetch_production( "fields": "datetime_beginning_ept,fuel_type,mw", "datetime_beginning_ept": target_datetime.strftime("%Y-%m-%dT%H:00:00.0000000"), } - resp_data = fetch_api_data(kind="gen_by_fuel", params=params) + resp_data = fetch_api_data(kind="gen_by_fuel", params=params, session=session) data = pd.DataFrame(resp_data) data.fuel_type = data.fuel_type.map(FUEL_MAPPING) @@ -180,7 +178,7 @@ def add_default_tz(timestamp): return modified_timestamp -def get_miso_exchange(session: Optional[Session] = None) -> tuple: +def get_miso_exchange(session: Session) -> tuple: """ Current exchange status between PJM and MISO. :return: tuple containing flow and timestamp. @@ -188,9 +186,8 @@ def get_miso_exchange(session: Optional[Session] = None) -> tuple: map_url = "http://pjm.com/markets-and-operations/interregional-map.aspx" - s = session or Session() - req = s.get(map_url) - soup = BeautifulSoup(req.content, "html.parser") + res: Response = session.get(map_url) + soup = BeautifulSoup(res.text, "html.parser") find_div = soup.find("div", {"id": "body_0_flow1", "class": "flow"}) @@ -217,7 +214,7 @@ def get_miso_exchange(session: Optional[Session] = None) -> tuple: return flow, dt_aware -def get_exchange_data(interface, session: Optional[Session] = None) -> list: +def get_exchange_data(interface, session: Session) -> list: """ This function can fetch 5min data for any PJM interface in the current day. Extracts load and timestamp data from html source then joins them together. @@ -226,9 +223,8 @@ def get_exchange_data(interface, session: Optional[Session] = None) -> list: base_url = "http://www.pjm.com/Charts/InterfaceChart.aspx?open=" url = base_url + exchange_mapping[interface] - s = session or Session() - req = s.get(url) - soup = BeautifulSoup(req.content, "html.parser") + res: Response = session.get(url) + soup = BeautifulSoup(res.text, "html.parser") scripts = soup.find( "script", @@ -265,16 +261,16 @@ def get_exchange_data(interface, session: Optional[Session] = None) -> list: return converted_flows -def combine_NY_exchanges() -> list: +def combine_NY_exchanges(session: Session) -> list: """ Combination function for the 4 New York interfaces. Timestamps are checked to ensure correct combination. """ - nyiso = get_exchange_data("nyiso", session=None) - neptune = get_exchange_data("neptune", session=None) - linden = get_exchange_data("linden", session=None) - hudson = get_exchange_data("hudson", session=None) + nyiso = get_exchange_data("nyiso", session) + neptune = get_exchange_data("neptune", session) + linden = get_exchange_data("linden", session) + hudson = get_exchange_data("hudson", session) combined_flows = zip(nyiso, neptune, linden, hudson) @@ -298,7 +294,7 @@ def combine_NY_exchanges() -> list: def fetch_exchange( zone_key1: str, zone_key2: str, - session: Optional[Session] = None, + session: Session = Session(), target_datetime: Optional[datetime] = None, logger: Logger = getLogger(__name__), ) -> Union[List[dict], dict]: @@ -310,12 +306,12 @@ def fetch_exchange( sortedcodes = "->".join(sorted([zone_key1, zone_key2])) if sortedcodes == "US-NY->US-PJM": - flows = combine_NY_exchanges() + flows = combine_NY_exchanges(session) elif sortedcodes == "US-MIDA-PJM->US-NY-NYIS": - flows = combine_NY_exchanges() + flows = combine_NY_exchanges(session) flows = [(-total, dt) for total, dt in flows] elif sortedcodes == "US-MISO->US-PJM": - flow = get_miso_exchange() + flow = get_miso_exchange(session) exchange = { "sortedZoneKeys": sortedcodes, "datetime": flow[1], @@ -324,7 +320,7 @@ def fetch_exchange( } return exchange elif sortedcodes == "US-MIDA-PJM->US-MIDW-MISO": - flow = get_miso_exchange() + flow = get_miso_exchange(session) exchange = { "sortedZoneKeys": sortedcodes, "datetime": flow[1], @@ -350,7 +346,7 @@ def fetch_exchange( def fetch_price( zone_key: str = "US-PJM", - session: Optional[Session] = None, + session: Session = Session(), target_datetime: Optional[datetime] = None, logger: Logger = getLogger(__name__), ) -> dict: @@ -358,9 +354,8 @@ def fetch_price( if target_datetime is not None: raise NotImplementedError("This parser is not yet able to parse past dates") - s = session or Session() - req = s.get(url) - soup = BeautifulSoup(req.content, "html.parser") + res: Response = session.get(url) + soup = BeautifulSoup(res.text, "html.parser") price_tag = soup.find("span", class_="rtolmpico") price_data = price_tag.find_next("h2") From 47cf49dde71ce627d7c62d8872ab85c2bb46123e Mon Sep 17 00:00:00 2001 From: Mathilde Daugy Date: Mon, 19 Dec 2022 10:49:21 +0100 Subject: [PATCH 14/15] added get_api_key --- parsers/US_PJM.py | 64 +++++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 5e15e8cf12..0a1aa587ad 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -13,7 +13,7 @@ import pytz from bs4 import BeautifulSoup from dateutil import parser, tz -from requests import Session, Response +from requests import Response, Session from parsers.lib.config import refetch_frequency from parsers.lib.exceptions import ParserException @@ -66,16 +66,28 @@ } +def get_api_subscription_key(session: Session) -> str: + pjm_settings: Response = session.get( + "https://dataminer2.pjm.com/config/settings.json" + ) + if pjm_settings.status_code == 200: + return pjm_settings.json()["subscriptionKey"] + raise ParserException( + parser="US_PJM.py", + message=f"Could not get API key", + ) + + def fetch_api_data(kind: str, params: dict, session: Session) -> list: headers = { "Host": "api.pjm.com", - "Ocp-Apim-Subscription-Key": get_token("PJM_TOKEN"), + "Ocp-Apim-Subscription-Key": get_api_subscription_key(session=session), "Origin": "http://dataminer2.pjm.com", "Referer": "http://dataminer2.pjm.com/", } url = API_ENDPOINT + kind - resp: Response = session.get(url, params, headers=headers) + resp: Response = session.get(url=url, params=params, headers=headers) if resp.status_code == 200: data = resp.json() return data @@ -99,8 +111,6 @@ def fetch_consumption_forecast_7_days( if not session: session = Session() - headers = {"Ocp-Apim-Subscription-Key": get_token("PJM_TOKEN")} - # startRow must be set if forecast_area is set. # RTO_COMBINED is area for whole PJM zone. params = {"download": True, "startRow": 1, "forecast_area": "RTO_COMBINED"} @@ -139,25 +149,25 @@ def fetch_production( "fields": "datetime_beginning_ept,fuel_type,mw", "datetime_beginning_ept": target_datetime.strftime("%Y-%m-%dT%H:00:00.0000000"), } - resp_data = fetch_api_data(kind="gen_by_fuel", params=params) + resp_data = fetch_api_data(kind="gen_by_fuel", params=params, session=session) data = pd.DataFrame(resp_data) + data.datetime_beginning_ept = pd.to_datetime(data.datetime_beginning_ept) + data = data.set_index("datetime_beginning_ept") data.fuel_type = data.fuel_type.map(FUEL_MAPPING) all_data_points = [] - for dt in set(data.datetime_beginning_ept): + for dt in set(data.index): production = {} storage = {} - data_dt = data.loc[data.datetime_beginning_ept == dt] - for mode in set(data.fuel_type): - if mode == "battery": - storage["battery"] = data_dt.loc[ - data_dt.fuel_type == mode, "mw" - ].values[0] + data_dt = data.loc[data.index == dt] + for i in range(len(data_dt)): + row = data_dt.iloc[i] + if row["fuel_type"] == "battery": + storage["battery"] = row.get("mw") else: - production[mode] = data_dt.loc[data_dt.fuel_type == mode, "mw"].values[ - 0 - ] + mode = row["fuel_type"] + production[mode] = row.get("mw") data_point = { "zoneKey": zone_key, "datetime": arrow.get(dt).datetime.replace( @@ -380,14 +390,14 @@ def fetch_price( return data -if __name__ == "__main__": - print("fetch_consumption_forecast_7_days() ->") - print(fetch_consumption_forecast_7_days()) - print("fetch_production() ->") - print(fetch_production()) - print("fetch_exchange(US-NY, US-PJM) ->") - print(fetch_exchange("US-NY", "US-PJM")) - print("fetch_exchange(US-MISO, US-PJM)") - print(fetch_exchange("US-MISO", "US-PJM")) - print("fetch_price() ->") - print(fetch_price()) +# if __name__ == "__main__": +# print("fetch_consumption_forecast_7_days() ->") +# print(fetch_consumption_forecast_7_days()) +# print("fetch_production() ->") +# print(fetch_production()) +# print("fetch_exchange(US-NY, US-PJM) ->") +# print(fetch_exchange("US-NY", "US-PJM")) +# print("fetch_exchange(US-MISO, US-PJM)") +# print(fetch_exchange("US-MISO", "US-PJM")) +# print("fetch_price() ->") +# print(fetch_price()) From 26ce68dfe0794053ef9ac7573fb7577b5332645c Mon Sep 17 00:00:00 2001 From: Mathilde Daugy Date: Mon, 19 Dec 2022 11:34:14 +0100 Subject: [PATCH 15/15] minor update to fetch_prod --- parsers/US_PJM.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/parsers/US_PJM.py b/parsers/US_PJM.py index 8ecef7e00e..7a7e4b04ea 100644 --- a/parsers/US_PJM.py +++ b/parsers/US_PJM.py @@ -17,7 +17,6 @@ from parsers.lib.config import refetch_frequency from parsers.lib.exceptions import ParserException -from parsers.lib.utils import get_token # Used for consumption forecast data. API_ENDPOINT = "https://api.pjm.com/api/v1/" @@ -157,7 +156,7 @@ def fetch_production( data.fuel_type = data.fuel_type.map(FUEL_MAPPING) all_data_points = [] - for dt in set(data.index): + for dt in data.index.unique(): production = {} storage = {} data_dt = data.loc[data.index == dt] @@ -387,14 +386,14 @@ def fetch_price( return data -# if __name__ == "__main__": -# print("fetch_consumption_forecast_7_days() ->") -# print(fetch_consumption_forecast_7_days()) -# print("fetch_production() ->") -# print(fetch_production()) -# print("fetch_exchange(US-NY, US-PJM) ->") -# print(fetch_exchange("US-NY", "US-PJM")) -# print("fetch_exchange(US-MISO, US-PJM)") -# print(fetch_exchange("US-MISO", "US-PJM")) -# print("fetch_price() ->") -# print(fetch_price()) +if __name__ == "__main__": + print("fetch_consumption_forecast_7_days() ->") + print(fetch_consumption_forecast_7_days()) + print("fetch_production() ->") + print(fetch_production()) + print("fetch_exchange(US-NY, US-PJM) ->") + print(fetch_exchange("US-NY", "US-PJM")) + print("fetch_exchange(US-MISO, US-PJM)") + print(fetch_exchange("US-MISO", "US-PJM")) + print("fetch_price() ->") + print(fetch_price())