Fix - Fixed default behavior for passed data, prepared new version

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [0.4.5] - 2023-03-10
+### Fix
+- Fixed default behavior for passed data.
+- Added a new graph for tune results.
+
 ## [0.4.4] - 2023-03-09
 ### Fix
 - Added missing possibility to set the method for load forecast to 'mlforecaster'.

setup.py

@@ -19,7 +19,7 @@
 
 setup(
     name='emhass',  # Required
-    version='0.4.4',  # Required
+    version='0.4.5',  # Required
     description='An Energy Management System for Home Assistant',  # Optional
     long_description=long_description,  # Optional
     long_description_content_type='text/markdown',  # Optional (see note above)

src/emhass/command_line.py

@@ -383,7 +383,8 @@ def forecast_model_predict(input_data_dict: dict, logger: logging.Logger,
     return predictions
 
 def forecast_model_tune(input_data_dict: dict, logger: logging.Logger,
-    debug: Optional[bool] = False, mlf: Optional[mlforecaster] = None) -> pd.DataFrame:
+    debug: Optional[bool] = False, mlf: Optional[mlforecaster] = None
+    ) -> Tuple[pd.DataFrame, mlforecaster]:
     """Tune a forecast model hyperparameters using bayesian optimization.
 
     :param input_data_dict: A dictionnary with multiple data used by the action functions
@@ -417,7 +418,7 @@ def forecast_model_tune(input_data_dict: dict, logger: logging.Logger,
         filename = model_type+'_mlf.pkl'
         with open(pathlib.Path(root) / filename, 'wb') as outp:
             pickle.dump(mlf, outp, pickle.HIGHEST_PROTOCOL)
-    return df_pred_optim
+    return df_pred_optim, mlf
 
 def publish_data(input_data_dict: dict, logger: logging.Logger,
     save_data_to_file: Optional[bool] = False, 
@@ -590,7 +591,7 @@ def main():
             _, _, mlf = forecast_model_fit(input_data_dict, logger, debug=args.debug)
         else:
             mlf = None
-        df_pred_optim = forecast_model_tune(input_data_dict, logger, debug=args.debug, mlf=mlf)
+        df_pred_optim, mlf = forecast_model_tune(input_data_dict, logger, debug=args.debug, mlf=mlf)
         opt_res = None
     elif args.action == 'publish-data':
         opt_res = publish_data(input_data_dict, logger)
@@ -609,7 +610,7 @@ def main():
     elif args.action == 'forecast-model-predict':
         return df_pred
     elif args.action == 'forecast-model-tune':
-        return df_pred_optim
+        return df_pred_optim, mlf
 
 if __name__ == '__main__':
     main()

src/emhass/utils.py

@@ -244,63 +244,62 @@ def treat_runtimeparams(runtimeparams: str, params: str, retrieve_hass_conf: dic
                     logger.warning("This value in prod_price_forecast was detected as non digits: "+str(x))
         else:
             params['passed_data']['prod_price_forecast'] = None
-        # Treat passed data for forecast model fit/predict/tune
-        if set_type == 'forecast-model-fit' or set_type == 'forecast-model-predict' or set_type == 'forecast-model-tune':
-            if 'days_to_retrieve' not in runtimeparams.keys():
-                days_to_retrieve = 30
-            else:
-                days_to_retrieve = runtimeparams['days_to_retrieve']
-            params['passed_data']['days_to_retrieve'] = days_to_retrieve
-            if 'model_type' not in runtimeparams.keys():
-                model_type = "load_forecast"
-            else:
-                model_type = runtimeparams['model_type']
-            params['passed_data']['model_type'] = model_type
-            if 'var_model' not in runtimeparams.keys():
-                var_model = "sensor.power_load_no_var_loads"
-            else:
-                var_model = runtimeparams['var_model']
-            params['passed_data']['var_model'] = var_model
-            if 'sklearn_model' not in runtimeparams.keys():
-                sklearn_model = "KNeighborsRegressor"
-            else:
-                sklearn_model = runtimeparams['sklearn_model']
-            params['passed_data']['sklearn_model'] = sklearn_model
-            if 'num_lags' not in runtimeparams.keys():
-                num_lags = 48
-            else:
-                num_lags = runtimeparams['num_lags']
-            params['passed_data']['num_lags'] = num_lags
-            if 'split_date_delta' not in runtimeparams.keys():
-                split_date_delta = '48h'
-            else:
-                split_date_delta = runtimeparams['split_date_delta']
-            params['passed_data']['split_date_delta'] = split_date_delta
-            if 'perform_backtest' not in runtimeparams.keys():
-                perform_backtest = False
-            else:
-                perform_backtest = runtimeparams['perform_backtest']
-            params['passed_data']['perform_backtest'] = perform_backtest
-            if 'model_predict_publish' not in runtimeparams.keys():
-                model_predict_publish = False
-            else:
-                model_predict_publish = runtimeparams['model_predict_publish']
-            params['passed_data']['model_predict_publish'] = model_predict_publish
-            if 'model_predict_entity_id' not in runtimeparams.keys():
-                model_predict_entity_id = "sensor.p_load_forecast_custom_model"
-            else:
-                model_predict_entity_id = runtimeparams['model_predict_entity_id']
-            params['passed_data']['model_predict_entity_id'] = model_predict_entity_id
-            if 'model_predict_unit_of_measurement' not in runtimeparams.keys():
-                model_predict_unit_of_measurement = "W"
-            else:
-                model_predict_unit_of_measurement = runtimeparams['model_predict_unit_of_measurement']
-            params['passed_data']['model_predict_unit_of_measurement'] = model_predict_unit_of_measurement
-            if 'model_predict_friendly_name' not in runtimeparams.keys():
-                model_predict_friendly_name = "Load Power Forecast custom ML model"
-            else:
-                model_predict_friendly_name = runtimeparams['model_predict_friendly_name']
-            params['passed_data']['model_predict_friendly_name'] = model_predict_friendly_name
+        # Treat passed data for forecast model fit/predict/tune at runtime
+        if 'days_to_retrieve' not in runtimeparams.keys():
+            days_to_retrieve = 30
+        else:
+            days_to_retrieve = runtimeparams['days_to_retrieve']
+        params['passed_data']['days_to_retrieve'] = days_to_retrieve
+        if 'model_type' not in runtimeparams.keys():
+            model_type = "load_forecast"
+        else:
+            model_type = runtimeparams['model_type']
+        params['passed_data']['model_type'] = model_type
+        if 'var_model' not in runtimeparams.keys():
+            var_model = "sensor.power_load_no_var_loads"
+        else:
+            var_model = runtimeparams['var_model']
+        params['passed_data']['var_model'] = var_model
+        if 'sklearn_model' not in runtimeparams.keys():
+            sklearn_model = "KNeighborsRegressor"
+        else:
+            sklearn_model = runtimeparams['sklearn_model']
+        params['passed_data']['sklearn_model'] = sklearn_model
+        if 'num_lags' not in runtimeparams.keys():
+            num_lags = 48
+        else:
+            num_lags = runtimeparams['num_lags']
+        params['passed_data']['num_lags'] = num_lags
+        if 'split_date_delta' not in runtimeparams.keys():
+            split_date_delta = '48h'
+        else:
+            split_date_delta = runtimeparams['split_date_delta']
+        params['passed_data']['split_date_delta'] = split_date_delta
+        if 'perform_backtest' not in runtimeparams.keys():
+            perform_backtest = False
+        else:
+            perform_backtest = runtimeparams['perform_backtest']
+        params['passed_data']['perform_backtest'] = perform_backtest
+        if 'model_predict_publish' not in runtimeparams.keys():
+            model_predict_publish = False
+        else:
+            model_predict_publish = runtimeparams['model_predict_publish']
+        params['passed_data']['model_predict_publish'] = model_predict_publish
+        if 'model_predict_entity_id' not in runtimeparams.keys():
+            model_predict_entity_id = "sensor.p_load_forecast_custom_model"
+        else:
+            model_predict_entity_id = runtimeparams['model_predict_entity_id']
+        params['passed_data']['model_predict_entity_id'] = model_predict_entity_id
+        if 'model_predict_unit_of_measurement' not in runtimeparams.keys():
+            model_predict_unit_of_measurement = "W"
+        else:
+            model_predict_unit_of_measurement = runtimeparams['model_predict_unit_of_measurement']
+        params['passed_data']['model_predict_unit_of_measurement'] = model_predict_unit_of_measurement
+        if 'model_predict_friendly_name' not in runtimeparams.keys():
+            model_predict_friendly_name = "Load Power Forecast custom ML model"
+        else:
+            model_predict_friendly_name = runtimeparams['model_predict_friendly_name']
+        params['passed_data']['model_predict_friendly_name'] = model_predict_friendly_name
         # Treat optimization configuration parameters passed at runtime 
         if 'num_def_loads' in runtimeparams.keys():
             optim_conf['num_def_loads'] = runtimeparams['num_def_loads']

src/emhass/web_server.py

@@ -31,15 +31,15 @@ def get_injection_dict(df, plot_size = 1366):
     # Create plots
     cols_p = [i for i in df.columns.to_list() if 'P_' in i]
     fig_0 = px.line(df[cols_p], title='Systems powers schedule after optimization results', 
-                    template='plotly_white', width=plot_size, height=0.5*plot_size, line_shape="hv")
+                    template='presentation', width=plot_size, height=0.5*plot_size, line_shape="hv")
     fig_0.update_layout(xaxis_title='Timestamp', yaxis_title='System powers (W)')
     if 'SOC_opt' in df.columns.to_list():
         fig_1 = px.line(df['SOC_opt'], title='Battery state of charge schedule after optimization results', 
-                        template='plotly_white', width=plot_size, height=0.5*plot_size, line_shape="hv")
+                        template='presentation', width=plot_size, height=0.5*plot_size, line_shape="hv")
         fig_1.update_layout(xaxis_title='Timestamp', yaxis_title='Battery SOC (%)')
     cols_cost = [i for i in df.columns.to_list() if 'cost_' in i or 'unit_' in i]
     fig_2 = px.line(df[cols_cost], title='Systems costs obtained from optimization results', 
-                      template='plotly_white', width=plot_size, height=0.5*plot_size, line_shape="hv")
+                      template='presentation', width=plot_size, height=0.5*plot_size, line_shape="hv")
     fig_2.update_layout(xaxis_title='Timestamp', yaxis_title='System costs (currency)')
     # Get full path to image
     image_path_0 = fig_0.to_html(full_html=False, default_width='75%')
@@ -66,7 +66,7 @@ def get_injection_dict(df, plot_size = 1366):
 
 def get_injection_dict_forecast_model_fit(df_fit_pred, mlf):
     fig = df_fit_pred.plot()
-    fig.layout.template = 'plotly_white'
+    fig.layout.template = 'presentation'
     fig.update_yaxes(title_text = mlf.model_type)
     fig.update_xaxes(title_text = "Time")
     image_path_0 = fig.to_html(full_html=False, default_width='75%')
@@ -78,6 +78,20 @@ def get_injection_dict_forecast_model_fit(df_fit_pred, mlf):
     injection_dict['figure_0'] = image_path_0
     return injection_dict
 
+def get_injection_dict_forecast_model_tune(df_pred_optim, mlf):
+    fig = df_pred_optim.plot()
+    fig.layout.template = 'presentation'
+    fig.update_yaxes(title_text = mlf.model_type)
+    fig.update_xaxes(title_text = "Time")
+    image_path_0 = fig.to_html(full_html=False, default_width='75%')
+    # The dict of plots
+    injection_dict = {}
+    injection_dict['title'] = '<h2>Custom machine learning forecast model tune</h2>'
+    injection_dict['subsubtitle0'] = '<h4>Performed a tuning routine using bayesian optimization for '+mlf.model_type+'</h4>'
+    injection_dict['subsubtitle0'] = '<h4>Forecasting variable '+mlf.var_model+'</h4>'
+    injection_dict['figure_0'] = image_path_0
+    return injection_dict
+
 def build_params(params, options, addon):
     if addon == 1:
         # Updating variables in retrieve_hass_conf
@@ -207,12 +221,9 @@ def action_call(action_name):
         return make_response(msg, 201)
     elif action_name == 'forecast-model-tune':
         app.logger.info(" >> Performing a machine learning forecast model tune...")
-        df_pred_optim = forecast_model_tune(input_data_dict, app.logger)
-        table1 = df_pred_optim.reset_index().to_html(classes='mystyle', index=False)
-        injection_dict = {}
-        injection_dict['title'] = '<h2>Custom machine learning forecast model tuning</h2>'
-        injection_dict['subsubtitle0'] = '<h4>Performed a tuning routine using bayesian optimization</h4>'
-        injection_dict['table1'] = table1
+        df_pred_optim, mlf = forecast_model_tune(input_data_dict, app.logger)
+        injection_dict = get_injection_dict_forecast_model_tune(
+            df_pred_optim, mlf)
         with open(str(data_path / 'injection_dict.pkl'), "wb") as fid:
             pickle.dump(injection_dict, fid)
         msg = f'EMHASS >> Action forecast-model-tune executed... \n'

tests/test_command_line_utils.py

@@ -283,8 +283,9 @@ def test_forecast_model_fit_predict_tune(self):
         self.assertIsInstance(df_pred, pd.Series)
         self.assertTrue(df_pred.isnull().sum().sum() == 0)
         # Test the tune method
-        df_pred_optim = forecast_model_tune(input_data_dict, logger, debug=True, mlf=mlf)
+        df_pred_optim, mlf = forecast_model_tune(input_data_dict, logger, debug=True, mlf=mlf)
         self.assertIsInstance(df_pred_optim, pd.DataFrame)
+        self.assertTrue(mlf.is_tuned == True)
 
     @patch('sys.argv', ['main', '--action', 'test', '--config', str(pathlib.Path(root+'/config_emhass.yaml')), 
                         '--debug', 'True'])
@@ -380,8 +381,9 @@ def test_main_forecast_model_tune(self):
         with patch('sys.argv', ['main', '--action', 'forecast-model-tune', '--config', str(pathlib.Path(root+'/config_emhass.yaml')), 
                                 '--params', params_json, '--runtimeparams', runtimeparams_json,
                                 '--debug', 'True']):
-            df_pred_optim = main()
+            df_pred_optim, mlf = main()
         self.assertIsInstance(df_pred_optim, pd.DataFrame)
+        self.assertTrue(mlf.is_tuned == True)
 
     @patch('sys.argv', ['main', '--action', 'publish-data', '--config', str(pathlib.Path(root+'/config_emhass.yaml')), 
                         '--debug', 'True'])