From 5ca132f63ca0c741bac767db48a61e0b699bc28e Mon Sep 17 00:00:00 2001 From: davidusb-geek Date: Tue, 24 Dec 2024 12:03:05 +0100 Subject: [PATCH 1/7] Added support to retrieve configuration data directly from HA, implemented currency and temperature units --- data/test_df_final.pkl | Bin 3678 -> 3955 bytes src/emhass/command_line.py | 39 ++++++++++++---------- src/emhass/utils.py | 30 ++++++++++++++--- tests/test_forecast.py | 8 ++--- tests/test_machine_learning_forecaster.py | 2 +- tests/test_optimization.py | 2 +- tests/test_retrieve_hass.py | 23 +++++++++++-- 7 files changed, 74 insertions(+), 30 deletions(-) diff --git a/data/test_df_final.pkl b/data/test_df_final.pkl index 5c07a96bf1fcaff2a344ead7a3a6de9d60cc7f7b..a6f4444a5a51845f799a3a988f9e94e7aa7b2e36 100644 GIT binary patch delta 303 zcmXwzO-chn5QR;mF_Q@j?tF%+AA`v7WpqAwcynuRy zY&?QHFX08OiCC+8kH>r8_tDSs;Cbg|zQ47((R-Tht)6IvyV6Dz+>)?;eofeu(K#@Z zb!U3RMgkQqL{Qey?DGJ775^a?Ko_T%x0CwytB*-#P5EygHrC`Glb|%@kJbR;U>cpR zVVnu4JYgD*3b|jdd;m=iM7IJ{hB-6bX~Ln9GU~`nP~^1(scIGe5U{N|uq_-22Gr{h z7Fu{NZIukqVeg~k$#%wpl>2LNB5bVA`t6LcyRcQ%oM>GQ3HuMq6uGYfil%SuDaJr2 Ezjs)7I{*Lx delta 24 ccmew?cTa}9fpuyy9|IU{ Tuple[str, dict]: """ Treat the passed optimization runtime parameters. @@ -183,6 +184,27 @@ def treat_runtimeparams( params["optim_conf"].update(optim_conf) params["plant_conf"].update(plant_conf) + # Check defaults on HA retrieved config + currency_to_symbol = { + 'EUR': '€', + 'USD': '$', + 'GBP': '£', + 'YEN': '¥', + 'JPY': '¥', + 'AUD': 'A$', + 'CAD': 'C$', + 'CHF': 'CHF', # Swiss Franc has no special symbol + 'CNY': '¥', + 'INR': '₹', + # Add more as needed + } + if 'currency' in ha_config.keys(): + ha_config['currency'] = currency_to_symbol.get(ha_config['currency'], 'Unknown') + else: + ha_config['currency'] = '€' + if 'unit_system' not in ha_config.keys(): + ha_config['unit_system'] = {'temperature': '°C'} + # Some default data needed custom_deferrable_forecast_id = [] custom_predicted_temperature_id = [] @@ -197,7 +219,7 @@ def treat_runtimeparams( custom_predicted_temperature_id.append( { "entity_id": "sensor.temp_predicted{}".format(k), - "unit_of_measurement": "°C", + "unit_of_measurement": ha_config['unit_system']['temperature'], "friendly_name": "Predicted temperature {}".format(k), } ) @@ -239,7 +261,7 @@ def treat_runtimeparams( }, "custom_cost_fun_id": { "entity_id": "sensor.total_cost_fun_value", - "unit_of_measurement": "", + "unit_of_measurement": ha_config['currency'], "friendly_name": "Total cost function value", }, "custom_optim_status_id": { @@ -249,12 +271,12 @@ def treat_runtimeparams( }, "custom_unit_load_cost_id": { "entity_id": "sensor.unit_load_cost", - "unit_of_measurement": "€/kWh", + "unit_of_measurement": f"{ha_config['currency']}/kWh", "friendly_name": "Unit Load Cost", }, "custom_unit_prod_price_id": { "entity_id": "sensor.unit_prod_price", - "unit_of_measurement": "€/kWh", + "unit_of_measurement": f"{ha_config['currency']}/kWh", "friendly_name": "Unit Prod Price", }, "custom_deferrable_forecast_id": custom_deferrable_forecast_id, diff --git a/tests/test_forecast.py b/tests/test_forecast.py index 38a370c5..fc1b9d6e 100644 --- a/tests/test_forecast.py +++ b/tests/test_forecast.py @@ -76,7 +76,7 @@ def setUp(self): # Obtain sensor values from saved file if self.get_data_from_file: with open(emhass_conf["data_path"] / "test_df_final.pkl", "rb") as inp: - self.rh.df_final, self.days_list, self.var_list = pickle.load(inp) + self.rh.df_final, self.days_list, self.var_list, self.rh.ha_config = pickle.load(inp) self.retrieve_hass_conf["sensor_power_load_no_var_loads"] = str( self.var_list[0] ) @@ -429,8 +429,8 @@ def test_get_forecasts_with_lists(self): ) # Obtain sensor values from saved file if self.get_data_from_file: - with open((emhass_conf["data_path"] / "test_df_final.pkl"), "rb") as inp: - rh.df_final, days_list, var_list = pickle.load(inp) + with open(emhass_conf["data_path"] / "test_df_final.pkl", "rb") as inp: + rh.df_final, days_list, var_list, rh.ha_config = pickle.load(inp) retrieve_hass_conf["sensor_power_load_no_var_loads"] = str(self.var_list[0]) retrieve_hass_conf["sensor_power_photovoltaics"] = str(self.var_list[1]) retrieve_hass_conf["sensor_linear_interp"] = [ @@ -667,7 +667,7 @@ def test_get_forecasts_with_lists_special_case(self): # Obtain sensor values from saved file if self.get_data_from_file: with open(emhass_conf["data_path"] / "test_df_final.pkl", "rb") as inp: - rh.df_final, days_list, var_list = pickle.load(inp) + rh.df_final, days_list, var_list, rh.ha_config = pickle.load(inp) retrieve_hass_conf["sensor_power_load_no_var_loads"] = str(self.var_list[0]) retrieve_hass_conf["sensor_power_photovoltaics"] = str(self.var_list[1]) retrieve_hass_conf["sensor_linear_interp"] = [ diff --git a/tests/test_machine_learning_forecaster.py b/tests/test_machine_learning_forecaster.py index ad77b8e9..43f7613d 100644 --- a/tests/test_machine_learning_forecaster.py +++ b/tests/test_machine_learning_forecaster.py @@ -100,7 +100,7 @@ def setUp(self): ) # Open and extract saved sensor data to test against with open(emhass_conf["data_path"] / "test_df_final.pkl", "rb") as inp: - self.rh.df_final, self.days_list, self.var_list = pickle.load(inp) + self.rh.df_final, self.days_list, self.var_list, self.rh.ha_config = pickle.load(inp) def test_fit(self): df_pred, df_pred_backtest = self.mlf.fit() diff --git a/tests/test_optimization.py b/tests/test_optimization.py index 733e93cb..5aa1d8cb 100644 --- a/tests/test_optimization.py +++ b/tests/test_optimization.py @@ -73,7 +73,7 @@ def setUp(self): # Obtain sensor values from saved file if get_data_from_file: with open(emhass_conf["data_path"] / "test_df_final.pkl", "rb") as inp: - self.rh.df_final, self.days_list, self.var_list = pickle.load(inp) + self.rh.df_final, self.days_list, self.var_list, self.rh.ha_config = pickle.load(inp) self.retrieve_hass_conf["sensor_power_load_no_var_loads"] = str( self.var_list[0] ) diff --git a/tests/test_retrieve_hass.py b/tests/test_retrieve_hass.py index 3034ab4a..30a59f5a 100644 --- a/tests/test_retrieve_hass.py +++ b/tests/test_retrieve_hass.py @@ -92,7 +92,7 @@ def setUp(self): # Obtain sensor values from saved file if self.get_data_from_file: with open(emhass_conf["data_path"] / "test_df_final.pkl", "rb") as inp: - self.rh.df_final, self.days_list, self.var_list = pickle.load(inp) + self.rh.df_final, self.days_list, self.var_list, self.rh.ha_config = pickle.load(inp) # Else obtain sensor values from HA else: self.days_list = get_days_list( @@ -108,11 +108,30 @@ def setUp(self): minimal_response=False, significant_changes_only=False, ) + # Mocking retrieve of ha_config using: self.rh.get_ha_config() + self.rh.ha_config = { + 'country': 'FR', + 'currency': 'EUR', + 'elevation': 4807, + 'latitude': 48.83, + 'longitude': 6.86, + 'time_zone': 'Europe/Paris', + 'unit_system': { + 'length': 'km', + 'accumulated_precipitation': 'mm', + 'area': 'm²', + 'mass': 'g', + 'pressure': 'Pa', + 'temperature': '°C', + 'volume': 'L', + 'wind_speed': 'm/s' + } + } # Check to save updated data to file if save_data_to_file: with open(emhass_conf["data_path"] / "test_df_final.pkl", "wb") as outp: pickle.dump( - (self.rh.df_final, self.days_list, self.var_list), + (self.rh.df_final, self.days_list, self.var_list, self.rh.ha_config), outp, pickle.HIGHEST_PROTOCOL, ) From fd31007a98e5970c055801c373146e65b43a4ef4 Mon Sep 17 00:00:00 2001 From: davidusb-geek Date: Wed, 25 Dec 2024 11:09:01 +0100 Subject: [PATCH 2/7] Fixing missing inputs to treat runtime params method --- src/emhass/forecast.py | 2 +- tests/test_command_line_utils.py | 1 + tests/test_forecast.py | 3 +++ tests/test_utils.py | 6 ++++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/emhass/forecast.py b/src/emhass/forecast.py index 8afd14ee..efb440b2 100644 --- a/src/emhass/forecast.py +++ b/src/emhass/forecast.py @@ -956,7 +956,7 @@ def get_load_forecast( if self.get_data_from_file: filename_path = self.emhass_conf["data_path"] / "test_df_final.pkl" with open(filename_path, "rb") as inp: - rh.df_final, days_list, var_list = pickle.load(inp) + rh.df_final, days_list, var_list, rh.ha_config = pickle.load(inp) self.var_load = var_list[0] self.retrieve_hass_conf["sensor_power_load_no_var_loads"] = ( self.var_load diff --git a/tests/test_command_line_utils.py b/tests/test_command_line_utils.py index e083c52f..d9341aa0 100644 --- a/tests/test_command_line_utils.py +++ b/tests/test_command_line_utils.py @@ -149,6 +149,7 @@ def test_set_input_data_dict(self): "load_power_forecast": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], "load_cost_forecast": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], "prod_price_forecast": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + "prediction_horizon": 10, } runtimeparams_json = json.dumps(runtimeparams) params = copy.deepcopy(json.loads(self.params_json)) diff --git a/tests/test_forecast.py b/tests/test_forecast.py index fc1b9d6e..fe16aba4 100644 --- a/tests/test_forecast.py +++ b/tests/test_forecast.py @@ -416,6 +416,7 @@ def test_get_forecasts_with_lists(self): set_type, logger, emhass_conf, + {}, ) # Build RetrieveHass Object rh = RetrieveHass( @@ -560,6 +561,7 @@ def test_get_forecasts_with_longer_lists(self): set_type, logger, emhass_conf, + {}, ) # Create Forecast Object fcst = Forecast( @@ -653,6 +655,7 @@ def test_get_forecasts_with_lists_special_case(self): set_type, logger, emhass_conf, + {}, ) # Create RetrieveHass Object rh = RetrieveHass( diff --git a/tests/test_utils.py b/tests/test_utils.py index d46051d0..e8b90411 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -162,6 +162,7 @@ def test_treat_runtimeparams(self): set_type, logger, emhass_conf, + {}, ) self.assertIsInstance(params, str) params = json.loads(params) @@ -184,6 +185,7 @@ def test_treat_runtimeparams(self): set_type, logger, emhass_conf, + {}, ) self.assertIsInstance(params, str) params = json.loads(params) @@ -249,6 +251,7 @@ def test_treat_runtimeparams(self): set_type, logger, emhass_conf, + {}, ) self.assertIsInstance(params, str) params = json.loads(params) @@ -344,6 +347,7 @@ def test_treat_runtimeparams_failed(self): set_type, logger, emhass_conf, + {}, ) self.assertTrue( @@ -402,6 +406,7 @@ def test_treat_runtimeparams_failed(self): set_type, logger, emhass_conf, + {}, ) self.assertIsInstance(runtimeparams["pv_power_forecast"], list) self.assertIsInstance(runtimeparams["load_power_forecast"], list) @@ -433,6 +438,7 @@ def test_treat_runtimeparams_failed(self): set_type, logger, emhass_conf, + {}, ) self.assertIsInstance(runtimeparams["pv_power_forecast"], str) self.assertIsInstance(runtimeparams["load_power_forecast"], str) From 54638b9b94b52fc42017a5bd17ab720440a9010b Mon Sep 17 00:00:00 2001 From: davidusb-geek Date: Thu, 26 Dec 2024 23:33:53 +0100 Subject: [PATCH 3/7] Changed to a new method to deal with HA config data --- src/emhass/command_line.py | 50 ++++++++++++++---------- src/emhass/utils.py | 79 ++++++++++++++++++++++++++------------ tests/test_utils.py | 37 +++++++++++++++--- 3 files changed, 116 insertions(+), 50 deletions(-) diff --git a/src/emhass/command_line.py b/src/emhass/command_line.py index fa3ec5bf..5b712c7e 100644 --- a/src/emhass/command_line.py +++ b/src/emhass/command_line.py @@ -69,8 +69,20 @@ def set_input_data_dict( retrieve_hass_conf, optim_conf, plant_conf = utils.get_yaml_parse(params, logger) if type(retrieve_hass_conf) is bool: return False - - # Define main objects + + # Treat runtimeparams + params, retrieve_hass_conf, optim_conf, plant_conf = utils.treat_runtimeparams( + runtimeparams, + params, + retrieve_hass_conf, + optim_conf, + plant_conf, + set_type, + logger, + emhass_conf, + ) + + # Define the data retrieve object rh = RetrieveHass( retrieve_hass_conf["hass_url"], retrieve_hass_conf["long_lived_token"], @@ -81,6 +93,21 @@ def set_input_data_dict( logger, get_data_from_file=get_data_from_file, ) + + # Retrieve basic configuration data from hass + if get_data_from_file: + with open(emhass_conf["data_path"] / "test_df_final.pkl", "rb") as inp: + _, _, _, rh.ha_config = pickle.load(inp) + else: + rh.get_ha_config() + + # Update the params dict using data from the HA configuration + params = utils.update_params_with_ha_config( + params, + rh.ha_config, + ) + + # Define the forecast and optimization objects fcst = Forecast( retrieve_hass_conf, optim_conf, @@ -100,24 +127,7 @@ def set_input_data_dict( emhass_conf, logger, ) - # Retrieve basic configuration data from hass - if get_data_from_file: - with open(emhass_conf["data_path"] / "test_df_final.pkl", "rb") as inp: - _, _, _, rh.ha_config = pickle.load(inp) - else: - rh.get_ha_config() - # Treat runtimeparams - params, retrieve_hass_conf, optim_conf, plant_conf = utils.treat_runtimeparams( - runtimeparams, - params, - retrieve_hass_conf, - optim_conf, - plant_conf, - set_type, - logger, - emhass_conf, - rh.ha_config, - ) + # Perform setup based on type of action if set_type == "perfect-optim": # Retrieve data from hass diff --git a/src/emhass/utils.py b/src/emhass/utils.py index 0a52855a..9c40f837 100644 --- a/src/emhass/utils.py +++ b/src/emhass/utils.py @@ -138,6 +138,55 @@ def get_forecast_dates( return forecast_dates +def update_params_with_ha_config( + params: str, + ha_config: dict, +) -> dict: + # Load serialized params + params = json.loads(params) + # Update params + currency_to_symbol = { + 'EUR': '€', + 'USD': '$', + 'GBP': '£', + 'YEN': '¥', + 'JPY': '¥', + 'AUD': 'A$', + 'CAD': 'C$', + 'CHF': 'CHF', # Swiss Franc has no special symbol + 'CNY': '¥', + 'INR': '₹', + # Add more as needed + } + if 'currency' in ha_config.keys(): + ha_config['currency'] = currency_to_symbol.get(ha_config['currency'], 'Unknown') + else: + ha_config['currency'] = '€' + if 'unit_system' not in ha_config.keys(): + ha_config['unit_system'] = {'temperature': '°C'} + + for k in range(params["optim_conf"]["number_of_deferrable_loads"]): + params['passed_data']['custom_predicted_temperature_id'][k].update( + {"unit_of_measurement": ha_config['unit_system']['temperature']} + ) + updated_passed_dict = { + "custom_cost_fun_id": { + "unit_of_measurement": ha_config['currency'], + }, + "custom_unit_load_cost_id": { + "unit_of_measurement": f"{ha_config['currency']}/kWh", + }, + "custom_unit_prod_price_id": { + "unit_of_measurement": f"{ha_config['currency']}/kWh", + }, + } + for key, value in updated_passed_dict.items(): + params["passed_data"][key]["unit_of_measurement"] = value["unit_of_measurement"] + # Serialize the final params + params = json.dumps(params, default=str) + return params + + def treat_runtimeparams( runtimeparams: str, params: str, @@ -147,7 +196,6 @@ def treat_runtimeparams( set_type: str, logger: logging.Logger, emhass_conf: dict, - ha_config: dict, ) -> Tuple[str, dict]: """ Treat the passed optimization runtime parameters. @@ -185,25 +233,8 @@ def treat_runtimeparams( params["plant_conf"].update(plant_conf) # Check defaults on HA retrieved config - currency_to_symbol = { - 'EUR': '€', - 'USD': '$', - 'GBP': '£', - 'YEN': '¥', - 'JPY': '¥', - 'AUD': 'A$', - 'CAD': 'C$', - 'CHF': 'CHF', # Swiss Franc has no special symbol - 'CNY': '¥', - 'INR': '₹', - # Add more as needed - } - if 'currency' in ha_config.keys(): - ha_config['currency'] = currency_to_symbol.get(ha_config['currency'], 'Unknown') - else: - ha_config['currency'] = '€' - if 'unit_system' not in ha_config.keys(): - ha_config['unit_system'] = {'temperature': '°C'} + default_currency_unit = '€' + default_temperature_unit = '°C' # Some default data needed custom_deferrable_forecast_id = [] @@ -219,7 +250,7 @@ def treat_runtimeparams( custom_predicted_temperature_id.append( { "entity_id": "sensor.temp_predicted{}".format(k), - "unit_of_measurement": ha_config['unit_system']['temperature'], + "unit_of_measurement": default_temperature_unit, "friendly_name": "Predicted temperature {}".format(k), } ) @@ -261,7 +292,7 @@ def treat_runtimeparams( }, "custom_cost_fun_id": { "entity_id": "sensor.total_cost_fun_value", - "unit_of_measurement": ha_config['currency'], + "unit_of_measurement": default_currency_unit, "friendly_name": "Total cost function value", }, "custom_optim_status_id": { @@ -271,12 +302,12 @@ def treat_runtimeparams( }, "custom_unit_load_cost_id": { "entity_id": "sensor.unit_load_cost", - "unit_of_measurement": f"{ha_config['currency']}/kWh", + "unit_of_measurement": f"{default_currency_unit}/kWh", "friendly_name": "Unit Load Cost", }, "custom_unit_prod_price_id": { "entity_id": "sensor.unit_prod_price", - "unit_of_measurement": f"{ha_config['currency']}/kWh", + "unit_of_measurement": f"{default_currency_unit}/kWh", "friendly_name": "Unit Prod Price", }, "custom_deferrable_forecast_id": custom_deferrable_forecast_id, diff --git a/tests/test_utils.py b/tests/test_utils.py index e8b90411..40d7a5e8 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -162,7 +162,6 @@ def test_treat_runtimeparams(self): set_type, logger, emhass_conf, - {}, ) self.assertIsInstance(params, str) params = json.loads(params) @@ -185,7 +184,6 @@ def test_treat_runtimeparams(self): set_type, logger, emhass_conf, - {}, ) self.assertIsInstance(params, str) params = json.loads(params) @@ -251,7 +249,6 @@ def test_treat_runtimeparams(self): set_type, logger, emhass_conf, - {}, ) self.assertIsInstance(params, str) params = json.loads(params) @@ -347,7 +344,6 @@ def test_treat_runtimeparams_failed(self): set_type, logger, emhass_conf, - {}, ) self.assertTrue( @@ -406,7 +402,6 @@ def test_treat_runtimeparams_failed(self): set_type, logger, emhass_conf, - {}, ) self.assertIsInstance(runtimeparams["pv_power_forecast"], list) self.assertIsInstance(runtimeparams["load_power_forecast"], list) @@ -438,13 +433,43 @@ def test_treat_runtimeparams_failed(self): set_type, logger, emhass_conf, - {}, ) self.assertIsInstance(runtimeparams["pv_power_forecast"], str) self.assertIsInstance(runtimeparams["load_power_forecast"], str) self.assertIsInstance(runtimeparams["load_cost_forecast"], str) self.assertIsInstance(runtimeparams["prod_price_forecast"], str) + def test_update_params_with_ha_config(self): + # Test dayahead runtime params + retrieve_hass_conf, optim_conf, plant_conf = utils.get_yaml_parse( + self.params_json, logger + ) + set_type = "dayahead-optim" + params, retrieve_hass_conf, optim_conf, plant_conf = utils.treat_runtimeparams( + self.runtimeparams_json, + self.params_json, + retrieve_hass_conf, + optim_conf, + plant_conf, + set_type, + logger, + emhass_conf, + ) + ha_config = { + 'currency': 'USD', + 'unit_system': {'temperature': '°F'} + } + params_json = utils.update_params_with_ha_config( + params, + ha_config, + ) + params = json.loads(params_json) + self.assertTrue(params["passed_data"]["custom_predicted_temperature_id"][0]["unit_of_measurement"] == "°F") + self.assertTrue(params["passed_data"]["custom_predicted_temperature_id"][1]["unit_of_measurement"] == "°F") + self.assertTrue(params["passed_data"]["custom_cost_fun_id"]["unit_of_measurement"] == '$') + self.assertTrue(params["passed_data"]["custom_unit_load_cost_id"]["unit_of_measurement"] == '$/kWh') + self.assertTrue(params["passed_data"]["custom_unit_prod_price_id"]["unit_of_measurement"] == '$/kWh') + def test_build_secrets(self): # Test the build_secrets defaults from get_test_params() params = TestCommandLineUtils.get_test_params() From 007815461a8d9c37fa671f18d3ba27bf417aec8d Mon Sep 17 00:00:00 2001 From: davidusb-geek Date: Thu, 26 Dec 2024 23:40:53 +0100 Subject: [PATCH 4/7] Solved number of args issue and iloc on series indexing FutureWarning --- src/emhass/optimization.py | 2 +- tests/test_forecast.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/emhass/optimization.py b/src/emhass/optimization.py index 3f58bfd4..acee4a15 100644 --- a/src/emhass/optimization.py +++ b/src/emhass/optimization.py @@ -661,7 +661,7 @@ def create_matrix(input_list, n): cooling_constant * ( predicted_temp[I - 1] - - outdoor_temperature_forecast[I - 1] + - outdoor_temperature_forecast.iloc[I - 1] ) ) ) diff --git a/tests/test_forecast.py b/tests/test_forecast.py index fe16aba4..fc1b9d6e 100644 --- a/tests/test_forecast.py +++ b/tests/test_forecast.py @@ -416,7 +416,6 @@ def test_get_forecasts_with_lists(self): set_type, logger, emhass_conf, - {}, ) # Build RetrieveHass Object rh = RetrieveHass( @@ -561,7 +560,6 @@ def test_get_forecasts_with_longer_lists(self): set_type, logger, emhass_conf, - {}, ) # Create Forecast Object fcst = Forecast( @@ -655,7 +653,6 @@ def test_get_forecasts_with_lists_special_case(self): set_type, logger, emhass_conf, - {}, ) # Create RetrieveHass Object rh = RetrieveHass( From 28d6acfb5d8a6ee2a23d4f38533b094d3e90c601 Mon Sep 17 00:00:00 2001 From: davidusb-geek Date: Fri, 27 Dec 2024 00:23:36 +0100 Subject: [PATCH 5/7] Added the new load forecast method --- docs/develop.md | 15 ++++ src/emhass/forecast.py | 74 ++++++++++++++++++- src/emhass/static/data/param_definitions.json | 3 +- src/emhass/utils.py | 15 ++++ tests/test_forecast.py | 13 ++++ 5 files changed, 116 insertions(+), 4 deletions(-) diff --git a/docs/develop.md b/docs/develop.md index 5d89a94d..9ee6c65b 100644 --- a/docs/develop.md +++ b/docs/develop.md @@ -318,6 +318,21 @@ Lastly, to support the configuration website to generate the parameter in the li ![Screenshot from 2024-09-09 16-45-32](https://github.com/user-attachments/assets/01e7984f-3332-4e25-8076-160f51a2e0c4) +If you are only adding another option for a existing parameter, editing param_definitions.json file should be all you need. (allowing the user to select the option from the configuration page): +```json +"load_forecast_method": { + "friendly_name": "Load forecast method", + "Description": "The load forecast method that will be used. The options are ‘csv’ to load a CSV file or ‘naive’ for a simple 1-day persistence model.", + "input": "select", + "select_options": [ + "naive", + "mlforecaster", + "csv", + "CALL_NEW_OPTION" + ], + "default_value": "naive" +}, +``` ## Step 3 - Pull request diff --git a/src/emhass/forecast.py b/src/emhass/forecast.py index efb440b2..76c2e56c 100644 --- a/src/emhass/forecast.py +++ b/src/emhass/forecast.py @@ -886,10 +886,47 @@ def get_forecast_out_from_csv_or_list( forecast_out = pd.concat([forecast_out, forecast_tp], axis=0) return forecast_out + @staticmethod + def get_typical_load_forecast(data, forecast_date): + """ + Forecast the load profile for the next day based on historic data. + + Parameters: + - data: pd.DataFrame with a DateTimeIndex containing the historic load data. Must include a 'load' column. + - forecast_date: pd.Timestamp for the date of the forecast. + - freq: frequency of the time series (e.g., '1H' for hourly). + + Returns: + - forecast: pd.Series with the forecasted load profile for the next day. + - used_days: list of days used to calculate the forecast. + """ + # Ensure the 'load' column exists + if 'load' not in data.columns: + raise ValueError("Data must have a 'load' column.") + # Filter historic data for the same month and day of the week + month = forecast_date.month + day_of_week = forecast_date.dayofweek + historic_data = data[(data.index.month == month) & (data.index.dayofweek == day_of_week)] + used_days = np.unique(historic_data.index.date) + # Align all historic data to the forecast day + aligned_data = [] + for day in used_days: + daily_data = data[data.index.date == pd.Timestamp(day).date()] + aligned_daily_data = daily_data.copy() + aligned_daily_data.index = aligned_daily_data.index.map( + lambda x: x.replace(year=forecast_date.year, month=forecast_date.month, day=forecast_date.day) + ) + aligned_data.append(aligned_daily_data) + # Combine all aligned historic data into a single DataFrame + combined_data = pd.concat(aligned_data) + # Compute the mean load for each timestamp + forecast = combined_data.groupby(combined_data.index).mean() + return forecast, used_days + def get_load_forecast( self, days_min_load_forecast: Optional[int] = 3, - method: Optional[str] = "naive", + method: Optional[str] = "typical", csv_path: Optional[str] = "data_load_forecast.csv", set_mix_forecast: Optional[bool] = False, df_now: Optional[pd.DataFrame] = pd.DataFrame(), @@ -904,10 +941,11 @@ def get_load_forecast( will be used to generate a naive forecast, defaults to 3 :type days_min_load_forecast: int, optional :param method: The method to be used to generate load forecast, the options \ + are 'typical' for a typical household load consumption curve, \ are 'naive' for a persistance model, 'mlforecaster' for using a custom \ previously fitted machine learning model, 'csv' to read the forecast from \ a CSV file and 'list' to use data directly passed at runtime as a list of \ - values. Defaults to 'naive'. + values. Defaults to 'typical'. :type method: str, optional :param csv_path: The path to the CSV file used when method = 'csv', \ defaults to "/data/data_load_forecast.csv" @@ -977,7 +1015,37 @@ def get_load_forecast( ): return False df = rh.df_final.copy()[[self.var_load_new]] - if method == "naive": # using a naive approach + if method == "typical": # using typical statistical data from a household power consumption + # Loading data from history file + model_type = "load_clustering" + data_path = self.emhass_conf["data_path"] / str("data_train_" + model_type + ".pkl") + with open(data_path, "rb") as fid: + data, _ = pickle.load(fid) + # Generate forecast + forecast_date = pd.Timestamp(self.forecast_dates[0].date()) # TODO: treat the case when forecast_dates spans more than 1 date + data.columns = ['load'] + forecast, used_days = Forecast.get_typical_load_forecast(data, forecast_date) # TODO: interpolate to time steps different than 30min + self.logger.debug(f"Using {len(used_days)} days of data to generate the forecast.") + # Normalize the forecast + forecast = forecast*self.plant_conf['maximum_power_from_grid']/9000 + data_list = forecast.values.ravel().tolist() # TODO: need to match the current date/time >> self.forecast_dates[0] + # Check if the passed data has the correct length + if ( + len(data_list) < len(self.forecast_dates) + and self.params["passed_data"]["prediction_horizon"] is None + ): + self.logger.error("Passed data from passed list is not long enough") + return False + else: + # Ensure correct length + data_list = data_list[0 : len(self.forecast_dates)] + # Define DataFrame + data_dict = {"ts": self.forecast_dates, "yhat": data_list} + data = pd.DataFrame.from_dict(data_dict) + # Define index + data.set_index("ts", inplace=True) + forecast_out = data.copy().loc[self.forecast_dates] + elif method == "naive": # using a naive approach mask_forecast_out = ( df.index > days_list[-1] - self.optim_conf["delta_forecast_daily"] ) diff --git a/src/emhass/static/data/param_definitions.json b/src/emhass/static/data/param_definitions.json index 9a03ff90..ecf7a599 100644 --- a/src/emhass/static/data/param_definitions.json +++ b/src/emhass/static/data/param_definitions.json @@ -101,11 +101,12 @@ "Description": "The load forecast method that will be used. The options are ‘csv’ to load a CSV file or ‘naive’ for a simple 1-day persistence model.", "input": "select", "select_options": [ + "typical", "naive", "mlforecaster", "csv" ], - "default_value": "naive" + "default_value": "typical" }, "set_total_pv_sell": { "friendly_name": "PV straight to grid", diff --git a/src/emhass/utils.py b/src/emhass/utils.py index 9c40f837..8c683cc4 100644 --- a/src/emhass/utils.py +++ b/src/emhass/utils.py @@ -142,6 +142,21 @@ def update_params_with_ha_config( params: str, ha_config: dict, ) -> dict: + """ + Update the params with the Home Assistant configuration. + + Parameters + ---------- + params : str + The serialized params. + ha_config : dict + The Home Assistant configuration. + + Returns + ------- + dict + The updated params. + """ # Load serialized params params = json.loads(params) # Update params diff --git a/tests/test_forecast.py b/tests/test_forecast.py index fc1b9d6e..6ead473f 100644 --- a/tests/test_forecast.py +++ b/tests/test_forecast.py @@ -901,6 +901,19 @@ def test_get_load_forecast_mlforecaster(self): self.assertTrue((P_load_forecast.index == self.fcst.forecast_dates).all()) self.assertEqual(len(self.P_PV_forecast), len(P_load_forecast)) + # Test load forecast with typical statistics method + def test_get_load_forecast_typical(self): + P_load_forecast = self.fcst.get_load_forecast(method='typical') + self.assertIsInstance(P_load_forecast, pd.core.series.Series) + self.assertIsInstance( + P_load_forecast.index, pd.core.indexes.datetimes.DatetimeIndex + ) + self.assertIsInstance( + P_load_forecast.index.dtype, pd.core.dtypes.dtypes.DatetimeTZDtype + ) + self.assertEqual(P_load_forecast.index.tz, self.fcst.time_zone) + self.assertEqual(len(self.P_PV_forecast), len(P_load_forecast)) + # Test load cost forecast dataframe output using saved csv referece file def test_get_load_cost_forecast(self): df_input_data = self.fcst.get_load_cost_forecast(self.df_input_data) From f04272b8d08efc700698af14f25fc6b94530ac55 Mon Sep 17 00:00:00 2001 From: davidusb-geek Date: Sat, 28 Dec 2024 20:32:03 +0100 Subject: [PATCH 6/7] Finished implementing the typical forecast method --- src/emhass/forecast.py | 90 ++++++++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/src/emhass/forecast.py b/src/emhass/forecast.py index 76c2e56c..f15a7574 100644 --- a/src/emhass/forecast.py +++ b/src/emhass/forecast.py @@ -887,18 +887,44 @@ def get_forecast_out_from_csv_or_list( return forecast_out @staticmethod - def get_typical_load_forecast(data, forecast_date): + def resample_data(data, freq, current_freq): + r""" + Resample a DataFrame with a custom frequency. + + :param data: Original time series data with a DateTimeIndex. + :type data: pd.DataFrame + :param freq: Desired frequency for resampling (e.g., pd.Timedelta("10min")). + :type freq: pd.Timedelta + :return: Resampled data at the specified frequency. + :rtype: pd.DataFrame """ + if freq > current_freq: + # Downsampling + # Use 'mean' to aggregate or choose other options ('sum', 'max', etc.) + resampled_data = data.resample(freq).mean() + elif freq < current_freq: + # Upsampling + # Use 'asfreq' to create empty slots, then interpolate + resampled_data = data.resample(freq).asfreq() + resampled_data = resampled_data.interpolate(method='time') + else: + # No resampling needed + resampled_data = data.copy() + return resampled_data + + @staticmethod + def get_typical_load_forecast(data, forecast_date): + r""" Forecast the load profile for the next day based on historic data. - - Parameters: - - data: pd.DataFrame with a DateTimeIndex containing the historic load data. Must include a 'load' column. - - forecast_date: pd.Timestamp for the date of the forecast. - - freq: frequency of the time series (e.g., '1H' for hourly). - - Returns: - - forecast: pd.Series with the forecasted load profile for the next day. - - used_days: list of days used to calculate the forecast. + + :param data: A DataFrame with a DateTimeIndex containing the historic load data. + Must include a 'load' column. + :type data: pd.DataFrame + :param forecast_date: The date for which the forecast will be generated. + :type forecast_date: pd.Timestamp + :return: A Series with the forecasted load profile for the next day and a list of days used + to calculate the forecast. + :rtype: tuple (pd.Series, list) """ # Ensure the 'load' column exists if 'load' not in data.columns: @@ -1021,30 +1047,28 @@ def get_load_forecast( data_path = self.emhass_conf["data_path"] / str("data_train_" + model_type + ".pkl") with open(data_path, "rb") as fid: data, _ = pickle.load(fid) + # Resample the data if needed + current_freq = pd.Timedelta('30min') + if self.freq != current_freq: + data = Forecast.resample_data(data, self.freq, current_freq) # Generate forecast - forecast_date = pd.Timestamp(self.forecast_dates[0].date()) # TODO: treat the case when forecast_dates spans more than 1 date - data.columns = ['load'] - forecast, used_days = Forecast.get_typical_load_forecast(data, forecast_date) # TODO: interpolate to time steps different than 30min - self.logger.debug(f"Using {len(used_days)} days of data to generate the forecast.") - # Normalize the forecast - forecast = forecast*self.plant_conf['maximum_power_from_grid']/9000 - data_list = forecast.values.ravel().tolist() # TODO: need to match the current date/time >> self.forecast_dates[0] - # Check if the passed data has the correct length - if ( - len(data_list) < len(self.forecast_dates) - and self.params["passed_data"]["prediction_horizon"] is None - ): - self.logger.error("Passed data from passed list is not long enough") - return False - else: - # Ensure correct length - data_list = data_list[0 : len(self.forecast_dates)] - # Define DataFrame - data_dict = {"ts": self.forecast_dates, "yhat": data_list} - data = pd.DataFrame.from_dict(data_dict) - # Define index - data.set_index("ts", inplace=True) - forecast_out = data.copy().loc[self.forecast_dates] + data_list = [] + dates_list = np.unique(self.forecast_dates.date).tolist() + forecast = pd.DataFrame() + for date in dates_list: + forecast_date = pd.Timestamp(date) + data.columns = ['load'] + forecast_tmp, used_days = Forecast.get_typical_load_forecast(data, forecast_date) + self.logger.debug(f"Using {len(used_days)} days of data to generate the forecast.") + # Normalize the forecast + forecast_tmp = forecast_tmp*self.plant_conf['maximum_power_from_grid']/9000 + data_list.extend(forecast_tmp.values.ravel().tolist()) + if len(forecast) == 0: + forecast = forecast_tmp + else: + forecast = pd.concat([forecast, forecast_tmp], axis=0) + forecast.index = forecast.index.tz_convert(self.time_zone) + forecast_out = forecast.loc[forecast.index.intersection(self.forecast_dates)] elif method == "naive": # using a naive approach mask_forecast_out = ( df.index > days_list[-1] - self.optim_conf["delta_forecast_daily"] From a1ef76b946480f353f7956f1b258ecf47b05de7e Mon Sep 17 00:00:00 2001 From: davidusb-geek Date: Sat, 28 Dec 2024 21:48:51 +0100 Subject: [PATCH 7/7] Fixed bad column and index name --- src/emhass/forecast.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/emhass/forecast.py b/src/emhass/forecast.py index f15a7574..361997f3 100644 --- a/src/emhass/forecast.py +++ b/src/emhass/forecast.py @@ -1069,6 +1069,8 @@ def get_load_forecast( forecast = pd.concat([forecast, forecast_tmp], axis=0) forecast.index = forecast.index.tz_convert(self.time_zone) forecast_out = forecast.loc[forecast.index.intersection(self.forecast_dates)] + forecast_out.index.name = 'ts' + forecast_out = forecast_out.rename(columns={'load': 'yhat'}) elif method == "naive": # using a naive approach mask_forecast_out = ( df.index > days_list[-1] - self.optim_conf["delta_forecast_daily"]