Merge pull request #196 from GeoDerp/#186

Dynamic tables & error logger feedback

.vscode/launch.json

@@ -2,15 +2,15 @@
   "configurations": [
     {
       "name": "Python: Current File",
-      "type": "python",
+      "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": true
     },
     {
       "name": "EMHASS run",
-      "type": "python",
+      "type": "debugpy",
       "request": "launch",
       "program": "web_server.py",
       "console": "integratedTerminal",
@@ -22,11 +22,12 @@
         "OPTIONS_PATH": "/workspaces/emhass/options.json",
         "SECRETS_PATH": "/workspaces/emhass/secrets_emhass.yaml",
         "DATA_PATH": "/workspaces/emhass/data/",
+        "LOGGING_LEVEL": "DEBUG"
       }
     },
     {
       "name": "EMHASS run ADDON",
-      "type": "python",
+      "type": "debugpy",
       "request": "launch",
       "program": "web_server.py",
       "console": "integratedTerminal",
@@ -45,6 +46,7 @@
         "LAT": "45.83", //optional change
         "LON": "6.86", //optional change
         "ALT": "4807.8", //optional change
+        "LOGGING_LEVEL": "DEBUG" //optional change
       },
 
   }

Add-onEmulateDocker

@@ -1,7 +1,9 @@
 #Test Add-on via Docker Emulation
-#Docker run example: 
-## docker build . -t emhass-addon -f AddonEmulateDocker && docker run  -e CONFIG_PATH="/usr/src/config_emhass.yaml" -e "SECRETS_PATH=/usr/src/secrets_emhass.yaml"  -e DATA_PATH="/data/" -e OPTIONS_PATH="/data/options.json" emhass-addon --url $URL --key $KEY
-FROM --platform=$BUILDPLATFORM ghcr.io/home-assistant/armhf-base-debian:bookworm AS build
+## Docker run example: 
+## docker build -t emhass/testing -f Add-onEmulateDocker .
+## docker run -it -p 5000:5000 --name emhass-test -e LAT="45.83" -e LON="6.86" -e ALT="4807.8" docker.io/emhass/testing --url YOURHAURLHERE --key YOURHAKEYHERE
+FROM ghcr.io/home-assistant/$TARGETARCH-base-debian:bookworm 
+
 WORKDIR /usr/src
 
 # copy the requirements file into the image

setup.py

@@ -62,5 +62,5 @@
             'emhass=emhass.command_line:main',
         ],
     },
-    package_data={'emhass': ['templates/index.html','static/style.css','static/img/emhass_icon.png','static/img/emhass_logo_short.svg']},
+    package_data={'emhass': ['templates/index.html','templates/template.html','static/style.css','static/img/emhass_icon.png','static/img/emhass_logo_short.svg']},
 )

src/emhass/command_line.py

@@ -75,12 +75,14 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
         else:
             days_list = utils.get_days_list(retrieve_hass_conf['days_to_retrieve'])
             var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']]
-            rh.get_data(days_list, var_list,
-                        minimal_response=False, significant_changes_only=False)
-        rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'],
+            if not rh.get_data(days_list, var_list,
+                        minimal_response=False, significant_changes_only=False):
+                return False 
+        if not rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'],
                         set_zero_min = retrieve_hass_conf['set_zero_min'], 
                         var_replace_zero = retrieve_hass_conf['var_replace_zero'], 
-                        var_interp = retrieve_hass_conf['var_interp'])
+                        var_interp = retrieve_hass_conf['var_interp']):
+            return False
         df_input_data = rh.df_final.copy()
         # What we don't need for this type of action
         P_PV_forecast, P_load_forecast, df_input_data_dayahead = None, None, None
@@ -89,6 +91,9 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
         df_weather = fcst.get_weather_forecast(method=optim_conf['weather_forecast_method'])
         P_PV_forecast = fcst.get_power_from_weather(df_weather)
         P_load_forecast = fcst.get_load_forecast(method=optim_conf['load_forecast_method'])
+        if isinstance(P_load_forecast,bool) and not P_load_forecast:
+            logger.error("Unable to get sensor_power_photovoltaics or sensor_power_load_no_var_loads")
+            return False
         df_input_data_dayahead = pd.DataFrame(np.transpose(np.vstack([P_PV_forecast.values,P_load_forecast.values])),
                                               index=P_PV_forecast.index,
                                               columns=['P_PV_forecast', 'P_load_forecast'])
@@ -107,12 +112,14 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
         else:
             days_list = utils.get_days_list(1)
             var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']]
-            rh.get_data(days_list, var_list,
-                        minimal_response=False, significant_changes_only=False)
-        rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'],
+            if not rh.get_data(days_list, var_list,
+                        minimal_response=False, significant_changes_only=False):
+                return False
+        if not rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'],
                         set_zero_min = retrieve_hass_conf['set_zero_min'], 
                         var_replace_zero = retrieve_hass_conf['var_replace_zero'], 
-                        var_interp = retrieve_hass_conf['var_interp'])
+                        var_interp = retrieve_hass_conf['var_interp']):
+            return False
         df_input_data = rh.df_final.copy()
         # Get PV and load forecasts
         df_weather = fcst.get_weather_forecast(method=optim_conf['weather_forecast_method'])
@@ -143,7 +150,8 @@ def set_input_data_dict(config_path: pathlib.Path, base_path: str, costfun: str,
         else:
             days_list = utils.get_days_list(days_to_retrieve)
             var_list = [var_model]
-            rh.get_data(days_list, var_list)
+            if not rh.get_data(days_list, var_list):
+                return False
             df_input_data = rh.df_final.copy()
     elif set_type == "publish-data":
         df_input_data, df_input_data_dayahead = None, None
@@ -415,7 +423,7 @@ def forecast_model_tune(input_data_dict: dict, logger: logging.Logger,
                 mlf = pickle.load(inp)
         else:
             logger.error("The ML forecaster file was not found, please run a model fit method before this tune method")
-            return
+            return None, None
     # Tune the model
     df_pred_optim = mlf.tune(debug=debug)
     # Save model
@@ -540,6 +548,9 @@ def publish_data(input_data_dict: dict, logger: logging.Logger,
                                     publish_prefix = publish_prefix)
     # Publish the optimization status
     custom_cost_fun_id = params['passed_data']['custom_optim_status_id']
+    if "optim_status" not in opt_res_latest:
+        opt_res_latest["optim_status"] = 'Optimal'
+        logger.warning("no optim_status in opt_res_latest, run an optimization task first")
     input_data_dict['rh'].post_data(opt_res_latest['optim_status'], idx_closest, 
                                     custom_cost_fun_id["entity_id"], 
                                     custom_cost_fun_id["unit_of_measurement"],

src/emhass/forecast.py

@@ -585,12 +585,14 @@ def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: O
                 with open(pathlib.Path(self.root) / 'data' / 'test_df_final.pkl', 'rb') as inp:
                     rh.df_final, days_list, _ = pickle.load(inp)
             else:
-                days_list = get_days_list(days_min_load_forecast)
-                rh.get_data(days_list, var_list)
-            rh.prepare_data(self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'],
+                days_list = get_days_list(days_min_load_forecast) 
+                if not rh.get_data(days_list, var_list):
+                    return False
+            if  not rh.prepare_data(self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'],
                             set_zero_min = self.retrieve_hass_conf['set_zero_min'], 
                             var_replace_zero = var_replace_zero, 
-                            var_interp = var_interp)
+                            var_interp = var_interp):
+                return False
             df = rh.df_final.copy()[[self.var_load_new]]
         if method == 'naive': # using a naive approach
             mask_forecast_out = (df.index > days_list[-1] - self.optim_conf['delta_forecast'])

src/emhass/retrieve_hass.py

@@ -92,6 +92,7 @@ def get_data(self, days_list: pd.date_range, var_list: list, minimal_response: O
         """
         self.logger.info("Retrieve hass get data method initiated...")
         self.df_final = pd.DataFrame()
+        x = 0 #iterate based on days
         # Looping on each day from days list
         for day in days_list:
 
@@ -115,8 +116,14 @@ def get_data(self, days_list: pd.date_range, var_list: list, minimal_response: O
                 try:
                     response = get(url, headers=headers)
                 except Exception:
-                    return "Request Get Error"
+                    self.logger.error("Unable to access Home Assistance instance, check URL")
+                    self.logger.error("If using addon, try setting url and token to 'empty'")
+                    return False
                 else:
+                    if response.status_code == 401:
+                        self.logger.error("Unable to access Home Assistance instance, TOKEN/KEY")
+                        self.logger.error("If using addon, try setting url and token to 'empty'")
+                        return False
                     if response.status_code > 299:
                         return f"Request Get Error: {response.status_code}"
                 '''import bz2 # Uncomment to save a serialized data for tests
@@ -126,13 +133,18 @@ def get_data(self, days_list: pd.date_range, var_list: list, minimal_response: O
                 try: # Sometimes when there are connection problems we need to catch empty retrieved json
                     data = response.json()[0]
                 except IndexError:
-                    self.logger.error("The retrieved JSON is empty, check that correct day or variable names are passed")
-                    self.logger.error("Either the names of the passed variables are not correct or days_to_retrieve is larger than the recorded history of your sensor (check your recorder settings)")
-                    break
+                    if x is 0:
+                        self.logger.error("The retrieved JSON is empty, A sensor:" + var + " may have 0 days of history or passed sensor may not be correct")
+                    else:
+                        self.logger.error("The retrieved JSON is empty, days_to_retrieve may be larger than the recorded history of sensor:" + var + "  (check your recorder settings)")
+                    return False
                 df_raw = pd.DataFrame.from_dict(data)
                 if len(df_raw) == 0:
-                    self.logger.error("Retrieved empty Dataframe, check that correct day or variable names are passed")
-                    self.logger.error("Either the names of the passed variables are not correct or days_to_retrieve is larger than the recorded history of your sensor (check your recorder settings)")
+                    if x is 0:
+                        self.logger.error("The retrieved Dataframe is empty, A sensor:" + var + " may have 0 days of history or passed sensor may not be correct")
+                    else:
+                        self.logger.error("Retrieved empty Dataframe, days_to_retrieve may be larger than the recorded history of sensor:" + var + "  (check your recorder settings)")
+                    return False
                 if i == 0: # Defining the DataFrame container
                     from_date = pd.to_datetime(df_raw['last_changed'], format="ISO8601").min()
                     to_date = pd.to_datetime(df_raw['last_changed'], format="ISO8601").max()
@@ -147,11 +159,13 @@ def get_data(self, days_list: pd.date_range, var_list: list, minimal_response: O
                 df_tp = df_tp.resample(self.freq).mean()
                 df_day = pd.concat([df_day, df_tp], axis=1)
 
+            x += 1
             self.df_final = pd.concat([self.df_final, df_day], axis=0)
         self.df_final = set_df_index_freq(self.df_final)
         if self.df_final.index.freq != self.freq:
             self.logger.error("The inferred freq from data is not equal to the defined freq in passed parameters")
-
+            return False
+        return True
 
     def prepare_data(self, var_load: str, load_negative: Optional[bool] = False, set_zero_min: Optional[bool] = True,
                      var_replace_zero: Optional[list] = None, var_interp: Optional[list] = None) -> None:
@@ -185,6 +199,10 @@ def prepare_data(self, var_load: str, load_negative: Optional[bool] = False, set
             self.df_final.drop([var_load], inplace=True, axis=1)
         except KeyError:
             self.logger.error("Variable "+var_load+" was not found. This is typically because no data could be retrieved from Home Assistant")
+            return False
+        except ValueError:
+            self.logger.error("sensor.power_photovoltaics and sensor.power_load_no_var_loads should not be the same")
+            return False   
         if set_zero_min: # Apply minimum values
             self.df_final.clip(lower=0.0, inplace=True, axis=1)
             self.df_final.replace(to_replace=0.0, value=np.nan, inplace=True)
@@ -215,6 +233,7 @@ def prepare_data(self, var_load: str, load_negative: Optional[bool] = False, set
             self.df_final.index = self.df_final.index.tz_convert(self.time_zone)
         # Drop datetimeindex duplicates on final DF
         self.df_final = self.df_final[~self.df_final.index.duplicated(keep='first')]
+        return True
 
     @staticmethod
     def get_attr_data_dict(data_df: pd.DataFrame, idx: int, entity_id: str, 

src/emhass/static/style.css

@@ -568,6 +568,10 @@ button {
   overflow-x: auto !important;
   margin: 0 auto;
   margin-bottom: 10px;
+  animation-name: fadeInOpacity;
+  animation-iteration-count: 1;
+  animation-timing-function: ease-in-out;
+  animation-duration: .8s;
 }
 
 /* Set min size for diagrams  */
@@ -688,7 +692,7 @@ th {
   font-size: 4.0em;
   animation-name: fadeInOpacity;
   animation-iteration-count: 1;
-  animation-timing-function: ease-in;
+  animation-timing-function: ease-in-out;
   animation-duration: .5s;
 }
 
@@ -699,7 +703,7 @@ th {
   font-size: 4.0em;
   animation-name: fadeInOpacity;
   animation-iteration-count: 1;
-  animation-timing-function: ease-in;
+  animation-timing-function: ease-in-out;
   animation-duration: .5s;
 }
 
@@ -741,7 +745,10 @@ th {
   margin: 0 auto;
   width: fit-content;
   height: fit-content;
-  transition: 0.3s;
+  animation-name: fadeInOpacity;
+  animation-iteration-count: 1;
+  animation-timing-function: ease-in-out;
+  animation-duration: .8s;
 }
 
 #alert-text {