diff --git a/src/emhass/command_line.py b/src/emhass/command_line.py index e6a7fd61..605c19e3 100644 --- a/src/emhass/command_line.py +++ b/src/emhass/command_line.py @@ -76,12 +76,12 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str, days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] if not rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False): + minimal_response=False, significant_changes_only=False): return False if not rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], - set_zero_min = retrieve_hass_conf['set_zero_min'], - var_replace_zero = retrieve_hass_conf['var_replace_zero'], - var_interp = retrieve_hass_conf['var_interp']): + set_zero_min = retrieve_hass_conf['set_zero_min'], + var_replace_zero = retrieve_hass_conf['var_replace_zero'], + var_interp = retrieve_hass_conf['var_interp']): return False df_input_data = rh.df_final.copy() # What we don't need for this type of action @@ -113,12 +113,12 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str, days_list = utils.get_days_list(1) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] if not rh.get_data(days_list, var_list, - minimal_response=False, significant_changes_only=False): + minimal_response=False, significant_changes_only=False): return False if not rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'], - set_zero_min = retrieve_hass_conf['set_zero_min'], - var_replace_zero = retrieve_hass_conf['var_replace_zero'], - var_interp = retrieve_hass_conf['var_interp']): + set_zero_min = retrieve_hass_conf['set_zero_min'], + var_replace_zero = retrieve_hass_conf['var_replace_zero'], + var_interp = retrieve_hass_conf['var_interp']): return False df_input_data = rh.df_final.copy() # Get PV and load forecasts @@ -201,9 +201,11 @@ def perfect_forecast_optim(input_data_dict: dict, logger: logging.Logger, # Load cost and prod price forecast df_input_data = input_data_dict['fcst'].get_load_cost_forecast( input_data_dict['df_input_data'], - method=input_data_dict['fcst'].optim_conf['load_cost_forecast_method']) + method=input_data_dict['fcst'].optim_conf['load_cost_forecast_method'], + list_and_perfect=True) df_input_data = input_data_dict['fcst'].get_prod_price_forecast( - df_input_data, method=input_data_dict['fcst'].optim_conf['prod_price_forecast_method']) + df_input_data, method=input_data_dict['fcst'].optim_conf['prod_price_forecast_method'], + list_and_perfect=True) opt_res = input_data_dict['opt'].perform_perfect_forecast_optim(df_input_data, input_data_dict['days_list']) # Save CSV file for analysis if save_data_to_file: diff --git a/src/emhass/forecast.py b/src/emhass/forecast.py index b30c4e18..9eb731bf 100644 --- a/src/emhass/forecast.py +++ b/src/emhass/forecast.py @@ -23,7 +23,7 @@ from emhass.retrieve_hass import RetrieveHass from emhass.machine_learning_forecaster import MLForecaster -from emhass.utils import get_days_list, get_root +from emhass.utils import get_days_list, get_root, set_df_index_freq class Forecast(object): @@ -487,8 +487,9 @@ def get_forecast_days_csv(self, timedelta_days: Optional[int] = 1) -> pd.date_ra forecast_dates_csv = forecast_dates_csv[0:self.params['passed_data']['prediction_horizon']] return forecast_dates_csv - def get_forecast_out_from_csv(self, df_final: pd.DataFrame, forecast_dates_csv: pd.date_range, - csv_path: str, data_list: Optional[list] = None) -> pd.DataFrame: + def get_forecast_out_from_csv_or_list(self, df_final: pd.DataFrame, forecast_dates_csv: pd.date_range, + csv_path: str, data_list: Optional[list] = None, + list_and_perfect: Optional[bool] = False) -> pd.DataFrame: r""" Get the forecast data as a DataFrame from a CSV file. @@ -506,34 +507,67 @@ def get_forecast_out_from_csv(self, df_final: pd.DataFrame, forecast_dates_csv: :rtype: pd.DataFrame """ - days_list = df_final.index.day.unique().tolist() if csv_path is None: data_dict = {'ts':forecast_dates_csv, 'yhat':data_list} df_csv = pd.DataFrame.from_dict(data_dict) df_csv.index = forecast_dates_csv df_csv.drop(['ts'], axis=1, inplace=True) + df_csv = set_df_index_freq(df_csv) + if list_and_perfect: + days_list = df_final.index.day.unique().tolist() + else: + days_list = df_csv.index.day.unique().tolist() else: load_csv_file_path = self.root + csv_path df_csv = pd.read_csv(load_csv_file_path, header=None, names=['ts', 'yhat']) df_csv.index = forecast_dates_csv df_csv.drop(['ts'], axis=1, inplace=True) + df_csv = set_df_index_freq(df_csv) + days_list = df_final.index.day.unique().tolist() forecast_out = pd.DataFrame() for day in days_list: - first_elm_index = [i for i, x in enumerate(df_final.index.day == day) if x][0] - last_elm_index = [i for i, x in enumerate(df_final.index.day == day) if x][-1] - fcst_index = pd.date_range(start=df_final.index[first_elm_index], - end=df_final.index[last_elm_index], - freq=df_final.index.freq) - first_hour = str(df_final.index[first_elm_index].hour)+":"+str(df_final.index[first_elm_index].minute) - last_hour = str(df_final.index[last_elm_index].hour)+":"+str(df_final.index[last_elm_index].minute) + if csv_path is None: + if list_and_perfect: + df_tmp = copy.deepcopy(df_final) + else: + df_tmp = copy.deepcopy(df_csv) + else: + df_tmp = copy.deepcopy(df_final) + first_elm_index = [i for i, x in enumerate(df_tmp.index.day == day) if x][0] + last_elm_index = [i for i, x in enumerate(df_tmp.index.day == day) if x][-1] + fcst_index = pd.date_range(start=df_tmp.index[first_elm_index], + end=df_tmp.index[last_elm_index], + freq=df_tmp.index.freq) + first_hour = str(df_tmp.index[first_elm_index].hour)+":"+str(df_tmp.index[first_elm_index].minute) + last_hour = str(df_tmp.index[last_elm_index].hour)+":"+str(df_tmp.index[last_elm_index].minute) if len(forecast_out) == 0: - forecast_out = pd.DataFrame( - df_csv.between_time(first_hour, last_hour).values, - index=fcst_index) + if csv_path is None: + if list_and_perfect: + forecast_out = pd.DataFrame( + df_csv.between_time(first_hour, last_hour).values, + index=fcst_index) + else: + forecast_out = pd.DataFrame( + df_csv.loc[fcst_index,:].between_time(first_hour, last_hour).values, + index=fcst_index) + else: + forecast_out = pd.DataFrame( + df_csv.between_time(first_hour, last_hour).values, + index=fcst_index) else: - forecast_tp = pd.DataFrame( - df_csv.between_time(first_hour, last_hour).values, - index=fcst_index) + if csv_path is None: + if list_and_perfect: + forecast_tp = pd.DataFrame( + df_csv.between_time(first_hour, last_hour).values, + index=fcst_index) + else: + forecast_tp = pd.DataFrame( + df_csv.loc[fcst_index,:].between_time(first_hour, last_hour).values, + index=fcst_index) + else: + forecast_tp = pd.DataFrame( + df_csv.between_time(first_hour, last_hour).values, + index=fcst_index) forecast_out = pd.concat([forecast_out, forecast_tp], axis=0) return forecast_out @@ -668,7 +702,8 @@ def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: O return P_Load_forecast def get_load_cost_forecast(self, df_final: pd.DataFrame, method: Optional[str] = 'hp_hc_periods', - csv_path: Optional[str] = "data_load_cost_forecast.csv") -> pd.DataFrame: + csv_path: Optional[str] = "data_load_cost_forecast.csv", + list_and_perfect: Optional[bool] = False) -> pd.DataFrame: r""" Get the unit cost for the load consumption based on multiple tariff \ periods. This is the cost of the energy from the utility in a vector \ @@ -698,7 +733,7 @@ def get_load_cost_forecast(self, df_final: pd.DataFrame, method: Optional[str] = df_final.loc[df_hp.index, self.var_load_cost] = self.optim_conf['load_cost_hp'] elif method == 'csv': forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0) - forecast_out = self.get_forecast_out_from_csv( + forecast_out = self.get_forecast_out_from_csv_or_list( df_final, forecast_dates_csv, csv_path) df_final[self.var_load_cost] = forecast_out elif method == 'list': # reading a list of values @@ -712,8 +747,8 @@ def get_load_cost_forecast(self, df_final: pd.DataFrame, method: Optional[str] = data_list = data_list[0:len(self.forecast_dates)] # Define the correct dates forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0) - forecast_out = self.get_forecast_out_from_csv( - df_final, forecast_dates_csv, None, data_list=data_list) + forecast_out = self.get_forecast_out_from_csv_or_list( + df_final, forecast_dates_csv, None, data_list=data_list, list_and_perfect=list_and_perfect) # Fill the final DF df_final[self.var_load_cost] = forecast_out else: @@ -722,7 +757,8 @@ def get_load_cost_forecast(self, df_final: pd.DataFrame, method: Optional[str] = return df_final def get_prod_price_forecast(self, df_final: pd.DataFrame, method: Optional[str] = 'constant', - csv_path: Optional[str] = "/data/data_prod_price_forecast.csv") -> pd.DataFrame: + csv_path: Optional[str] = "/data/data_prod_price_forecast.csv", + list_and_perfect: Optional[bool] = False) -> pd.DataFrame: r""" Get the unit power production price for the energy injected to the grid.\ This is the price of the energy injected to the utility in a vector \ @@ -747,7 +783,7 @@ def get_prod_price_forecast(self, df_final: pd.DataFrame, method: Optional[str] df_final[self.var_prod_price] = self.optim_conf['prod_sell_price'] elif method == 'csv': forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0) - forecast_out = self.get_forecast_out_from_csv(df_final, + forecast_out = self.get_forecast_out_from_csv_or_list(df_final, forecast_dates_csv, csv_path) df_final[self.var_prod_price] = forecast_out @@ -762,8 +798,8 @@ def get_prod_price_forecast(self, df_final: pd.DataFrame, method: Optional[str] data_list = data_list[0:len(self.forecast_dates)] # Define the correct dates forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0) - forecast_out = self.get_forecast_out_from_csv( - df_final, forecast_dates_csv, None, data_list=data_list) + forecast_out = self.get_forecast_out_from_csv_or_list( + df_final, forecast_dates_csv, None, data_list=data_list, list_and_perfect=list_and_perfect) # Fill the final DF df_final[self.var_prod_price] = forecast_out else: diff --git a/tests/test_forecast.py b/tests/test_forecast.py index 2b27243e..2c259823 100644 --- a/tests/test_forecast.py +++ b/tests/test_forecast.py @@ -13,12 +13,12 @@ from emhass.machine_learning_forecaster import MLForecaster from emhass.forecast import Forecast from emhass.optimization import Optimization -from emhass.utils import get_root, get_logger, get_yaml_parse, treat_runtimeparams, get_days_list +from emhass import utils # the root folder -root = str(get_root(__file__, num_parent=2)) +root = str(utils.get_root(__file__, num_parent=2)) # create logger -logger, ch = get_logger(__name__, root, save_to_file=False) +logger, ch = utils.get_logger(__name__, root, save_to_file=False) class TestForecast(unittest.TestCase): @@ -41,7 +41,7 @@ def get_test_params(): def setUp(self): self.get_data_from_file = True params = None - retrieve_hass_conf, optim_conf, plant_conf = get_yaml_parse(pathlib.Path(root) / 'config_emhass.yaml', use_secrets=False) + retrieve_hass_conf, optim_conf, plant_conf = utils.get_yaml_parse(pathlib.Path(root) / 'config_emhass.yaml', use_secrets=False) self.retrieve_hass_conf, self.optim_conf, self.plant_conf = \ retrieve_hass_conf, optim_conf, plant_conf self.rh = RetrieveHass(self.retrieve_hass_conf['hass_url'], self.retrieve_hass_conf['long_lived_token'], @@ -51,7 +51,7 @@ def setUp(self): with open(pathlib.Path(root) / 'data' / 'test_df_final.pkl', 'rb') as inp: self.rh.df_final, self.days_list, self.var_list = pickle.load(inp) else: - self.days_list = get_days_list(self.retrieve_hass_conf['days_to_retrieve']) + self.days_list = utils.get_days_list(self.retrieve_hass_conf['days_to_retrieve']) self.var_list = [self.retrieve_hass_conf['var_load'], self.retrieve_hass_conf['var_PV']] self.rh.get_data(self.days_list, self.var_list, minimal_response=False, significant_changes_only=False) @@ -192,10 +192,10 @@ def test_get_forecasts_with_lists(self): runtimeparams_json = json.dumps(runtimeparams) params['passed_data'] = runtimeparams params_json = json.dumps(params) - retrieve_hass_conf, optim_conf, plant_conf = get_yaml_parse(pathlib.Path(root+'/config_emhass.yaml'), + retrieve_hass_conf, optim_conf, plant_conf = utils.get_yaml_parse(pathlib.Path(root+'/config_emhass.yaml'), use_secrets=False, params=params_json) set_type = "dayahead-optim" - params, retrieve_hass_conf, optim_conf, plant_conf = treat_runtimeparams( + params, retrieve_hass_conf, optim_conf, plant_conf = utils.treat_runtimeparams( runtimeparams_json, params_json, retrieve_hass_conf, optim_conf, plant_conf, set_type, logger) rh = RetrieveHass(retrieve_hass_conf['hass_url'], retrieve_hass_conf['long_lived_token'], @@ -205,7 +205,7 @@ def test_get_forecasts_with_lists(self): with open(pathlib.Path(root+'/data/test_df_final.pkl'), 'rb') as inp: rh.df_final, days_list, var_list = pickle.load(inp) else: - days_list = get_days_list(retrieve_hass_conf['days_to_retrieve']) + days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False) @@ -259,22 +259,22 @@ def test_get_forecasts_with_lists(self): } }) runtimeparams = { - 'pv_power_forecast':[i+1 for i in range(2*48)], - 'load_power_forecast':[i+1 for i in range(2*48)], - 'load_cost_forecast':[i+1 for i in range(2*48)], - 'prod_price_forecast':[i+1 for i in range(2*48)] + 'pv_power_forecast':[i+1 for i in range(3*48)], + 'load_power_forecast':[i+1 for i in range(3*48)], + 'load_cost_forecast':[i+1 for i in range(3*48)], + 'prod_price_forecast':[i+1 for i in range(3*48)] } runtimeparams_json = json.dumps(runtimeparams) params['passed_data'] = runtimeparams params_json = json.dumps(params) - retrieve_hass_conf, optim_conf, plant_conf = get_yaml_parse(pathlib.Path(root+'/config_emhass.yaml'), - use_secrets=False, params=params_json) - optim_conf['delta_forecast'] = pd.Timedelta(days=2) - params, retrieve_hass_conf, optim_conf, plant_conf = treat_runtimeparams( + retrieve_hass_conf, optim_conf, plant_conf = utils.get_yaml_parse(pathlib.Path(root+'/config_emhass.yaml'), + use_secrets=False, params=params_json) + optim_conf['delta_forecast'] = pd.Timedelta(days=3) + params, retrieve_hass_conf, optim_conf, plant_conf = utils.treat_runtimeparams( runtimeparams_json, params_json, retrieve_hass_conf, optim_conf, plant_conf, set_type, logger) fcst = Forecast(retrieve_hass_conf, optim_conf, plant_conf, - params_json, root, logger, opt_time_delta=48, get_data_from_file=True) + params_json, root, logger, get_data_from_file=True) P_PV_forecast = fcst.get_weather_forecast(method='list') self.assertIsInstance(P_PV_forecast, type(pd.DataFrame())) self.assertIsInstance(P_PV_forecast.index, pd.core.indexes.datetimes.DatetimeIndex) @@ -282,7 +282,7 @@ def test_get_forecasts_with_lists(self): self.assertEqual(P_PV_forecast.index.tz, fcst.time_zone) self.assertTrue(fcst.start_forecast < ts for ts in P_PV_forecast.index) self.assertTrue(P_PV_forecast.values[0][0] == 1) - self.assertTrue(P_PV_forecast.values[-1][0] == 2*48) + self.assertTrue(P_PV_forecast.values[-1][0] == 3*48) P_load_forecast = fcst.get_load_forecast(method='list') self.assertIsInstance(P_load_forecast, pd.core.series.Series) self.assertIsInstance(P_load_forecast.index, pd.core.indexes.datetimes.DatetimeIndex) @@ -290,7 +290,20 @@ def test_get_forecasts_with_lists(self): self.assertEqual(P_load_forecast.index.tz, fcst.time_zone) self.assertEqual(len(P_PV_forecast), len(P_load_forecast)) self.assertTrue(P_load_forecast.values[0] == 1) - self.assertTrue(P_load_forecast.values[-1] == 2*48) + self.assertTrue(P_load_forecast.values[-1] == 3*48) + df_input_data_dayahead = pd.concat([P_PV_forecast, P_load_forecast], axis=1) + df_input_data_dayahead = utils.set_df_index_freq(df_input_data_dayahead) + df_input_data_dayahead.columns = ['P_PV_forecast', 'P_load_forecast'] + df_input_data_dayahead = fcst.get_load_cost_forecast(df_input_data_dayahead, method='list') + self.assertTrue(fcst.var_load_cost in df_input_data_dayahead.columns) + self.assertTrue(df_input_data_dayahead.isnull().sum().sum()==0) + self.assertTrue(df_input_data_dayahead[fcst.var_load_cost].iloc[0] == 1) + self.assertTrue(df_input_data_dayahead[fcst.var_load_cost].iloc[-1] == 3*48) + df_input_data_dayahead = fcst.get_prod_price_forecast(df_input_data_dayahead, method='list') + self.assertTrue(fcst.var_prod_price in df_input_data_dayahead.columns) + self.assertTrue(df_input_data_dayahead.isnull().sum().sum()==0) + self.assertTrue(df_input_data_dayahead[fcst.var_prod_price].iloc[0] == 1) + self.assertTrue(df_input_data_dayahead[fcst.var_prod_price].iloc[-1] == 3*48) def test_get_forecasts_with_lists_special_case(self): with open(root+'/config_emhass.yaml', 'r') as file: @@ -312,10 +325,10 @@ def test_get_forecasts_with_lists_special_case(self): runtimeparams_json = json.dumps(runtimeparams) params['passed_data'] = runtimeparams params_json = json.dumps(params) - retrieve_hass_conf, optim_conf, plant_conf = get_yaml_parse(pathlib.Path(root+'/config_emhass.yaml'), + retrieve_hass_conf, optim_conf, plant_conf = utils.get_yaml_parse(pathlib.Path(root+'/config_emhass.yaml'), use_secrets=False, params=params_json) set_type = "dayahead-optim" - params, retrieve_hass_conf, optim_conf, plant_conf = treat_runtimeparams( + params, retrieve_hass_conf, optim_conf, plant_conf = utils.treat_runtimeparams( runtimeparams_json, params_json, retrieve_hass_conf, optim_conf, plant_conf, set_type, logger) rh = RetrieveHass(retrieve_hass_conf['hass_url'], retrieve_hass_conf['long_lived_token'], @@ -325,7 +338,7 @@ def test_get_forecasts_with_lists_special_case(self): with open(pathlib.Path(root+'/data/test_df_final.pkl'), 'rb') as inp: rh.df_final, days_list, var_list = pickle.load(inp) else: - days_list = get_days_list(retrieve_hass_conf['days_to_retrieve']) + days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve']) var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']] rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False)