Merge branch 'main' into image-persist

blueprints/event_summary.yaml

@@ -2,7 +2,7 @@ blueprint:
   name: AI Event Summary (LLM Vision v1.3.1)
   author: valentinfrlch
   description: >
-    AI-powered security event summaries for frigate or camera entities. 
+    AI-powered security event summaries for frigate or camera entities.
     Sends a notification with a preview to your phone that is updated dynamically when the AI summary is available.
   domain: automation
   source_url: https://github.com/valentinfrlch/ha-llmvision/blob/main/blueprints/event_summary.yaml
@@ -42,23 +42,43 @@ blueprint:
             integration: mobile_app
     camera_entities:
       name: Camera Entities
-      description: (Camera and Frigate mode) List of camera entities to monitor
+      description: >-
+        (Camera and Frigate mode)
+
+        List of camera entities to monitor
       default: []
       selector:
         entity:
           multiple: true
           filter:
             domain: camera
+    object_type:
+      name: Included Object Type(s)
+      description: >-
+        (Frigate mode only)
+
+        Only run if frigate labels the object as one of these. (person, dog, bird, etc)
+      default: []
+      selector:
+        text:
+          multiline: false
+          multiple: true
     trigger_state:
       name: Trigger State
-      description: (Camera mode only) Trigger the automation when your cameras change to this state.
+      description: >-
+        (Camera mode only)
+
+        Trigger the automation when your cameras change to this state.
       default: 'recording'
       selector:
         text:
           multiline: false
     motion_sensors:
       name: Motion Sensor
-      description: (Camera mode only) Set if your cameras don't change state. Use the same order used for camera entities.
+      description: >-
+        (Camera mode only)
+
+        Set if your cameras don't change state. Use the same order used for camera entities.
       default: []
       selector:
         entity:
@@ -67,7 +87,10 @@ blueprint:
             domain: binary_sensor
     preview_mode:
       name: Preview Mode
-      description: (Camera mode only) Choose between a live preview or a snapshot of the event
+      description: >-
+        (Camera mode only)
+
+        Choose between a live preview or a snapshot of the event
       default: 'Live Preview'
       selector:
         select:
@@ -84,22 +107,32 @@ blueprint:
           max: 60
     tap_navigate:
       name: Tap Navigate
-      description: Path to navigate to when notification is opened (e.g. /lovelace/cameras)
+      description: >-
+        Path to navigate to when notification is opened (e.g. /lovelace/cameras).
+
+        To have use the same input which was sent to the ai engine, use
+          `{{video if video != '''' else image}}`
       default: "/lovelace/0"
       selector:
         text:
           multiline: false
     duration:
       name: Duration
-      description: (Camera mode only) How long to record before analyzing (in seconds)
+      description: >-
+        (Camera mode only)
+
+        How long to record before analyzing (in seconds)
       default: 5
       selector:
         number:
           min: 1
           max: 60
     max_frames:
       name: Max Frames
-      description: (Camera and Frigate mode) How many frames to analyze. Picks frames with the most movement.
+      description: >-
+        (Camera and Frigate mode)
+
+        How many frames to analyze. Picks frames with the most movement.
       default: 3
       selector:
         number:
@@ -170,11 +203,12 @@ variables:
     {% set ns = namespace(device_names=[]) %}
     {% for device_id in notify_devices %}
       {% set device_name = device_attr(device_id, "name") %}
-      {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[^a-z0-9_\- ]", "") | replace(" ", "_") | replace("-", "_") %}
+      {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[' -]", "_") | regex_replace("[^a-z0-9_]", "")  %}
       {% set ns.device_names = ns.device_names + [sanitized_name] %}
     {% endfor %}
     {{ ns.device_names }}
   camera_entities_list: !input camera_entities
+  object_types_list: !input object_type
   motion_sensors_list: !input motion_sensors
   camera_entity: >
     {% if mode == 'Camera' %}
@@ -230,6 +264,10 @@ variables:
     Use "critical" only for possible burglaries and similar events. "time-sensitive" could be a courier at the front door or an event of similar importance.
     Reply with these replies exactly.
 
+max_exceeded: silent
+
+mode: single
+
 trigger:
   - platform: mqtt
     topic: "frigate/events"
@@ -247,9 +285,12 @@ condition:
   - condition: template
     value_template: >
       {% if mode == 'Frigate' %}
-        {{ trigger.payload_json["type"] == "end" and (state_attr(this.entity_id, 'last_triggered') is none or (now() - state_attr(this.entity_id, 'last_triggered')).total_seconds() / 60 > cooldown) and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list }}
-      {% else %}
-        {{ state_attr(this.entity_id, 'last_triggered') is none or (now() - state_attr(this.entity_id, 'last_triggered')).total_seconds() / 60 > cooldown }}
+        {{ trigger.payload_json["type"] == "end"
+           and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list
+           and ((object_types_list|length) == 0 or ((trigger.payload_json['after']['label']|lower) in object_types_list))
+        }}
+      {%else%}
+        true
       {% endif %}
 
 
@@ -293,7 +334,7 @@ action:
                       max_tokens: 3
                       temperature: 0.1
                     response_variable: importance
-    
+
   # Cancel automation if event not deemed important
   - choose:
       - conditions:
@@ -365,7 +406,7 @@ action:
               temperature: !input temperature
               expose_images: "{{true if preview_mode == 'Snapshot'}}"
             response_variable: response
-            
+
 
   - choose:
         - conditions:
@@ -389,3 +430,5 @@ action:
                         tag: "{{tag}}"
                         group: "{{group}}"
                         interruption-level: passive
+
+  - delay: '00:{{cooldown|int}}:00'

custom_components/llmvision/__init__.py

@@ -24,6 +24,8 @@
     IMAGE_ENTITY,
     VIDEO_FILE,
     EVENT_ID,
+    FRIGATE_RETRY_ATTEMPTS,
+    FRIGATE_RETRY_SECONDS,
     INTERVAL,
     DURATION,
     MAX_FRAMES,
@@ -228,6 +230,8 @@ def __init__(self, data_call):
             "\n") if data_call.data.get(EVENT_ID) else None
         self.interval = int(data_call.data.get(INTERVAL, 2))
         self.duration = int(data_call.data.get(DURATION, 10))
+        self.frigate_retry_attempts = int(data_call.data.get(FRIGATE_RETRY_ATTEMPTS, 2))
+        self.frigate_retry_seconds = int(data_call.data.get(FRIGATE_RETRY_SECONDS, 1))
         self.max_frames = int(data_call.data.get(MAX_FRAMES, 3))
         self.target_width = data_call.data.get(TARGET_WIDTH, 3840)
         self.temperature = float(data_call.data.get(TEMPERATURE, 0.3))
@@ -288,7 +292,9 @@ async def video_analyzer(data_call):
                                             target_width=call.target_width,
                                             include_filename=call.include_filename,
                                             expose_images=call.expose_images,
-                                            expose_images_persist=call.expose_images_persist
+                                            expose_images_persist=call.expose_images_persist,
+                                            frigate_retry_attempts=call.frigate_retry_attempts,
+                                            frigate_retry_seconds=call.frigate_retry_seconds
                                             )
         response = await client.make_request(call)
         await _remember(hass, call, start, response)

custom_components/llmvision/const.py

@@ -31,6 +31,8 @@
 EVENT_ID = 'event_id'
 INTERVAL = 'interval'
 DURATION = 'duration'
+FRIGATE_RETRY_ATTEMPTS = 'frigate_retry_attempts'
+FRIGATE_RETRY_SECONDS = 'frigate_retry_seconds'
 MAX_FRAMES = 'max_frames'
 DETAIL = 'detail'
 TEMPERATURE = 'temperature'

custom_components/llmvision/media_handlers.py

@@ -294,7 +294,7 @@ async def add_images(self, image_entities, image_paths, target_width, include_fi
                     raise ServiceValidationError(f"Error: {e}")
         return self.client
 
-    async def add_videos(self, video_paths, event_ids, max_frames, target_width, include_filename, expose_images, expose_images_persist):
+    async def add_videos(self, video_paths, event_ids, max_frames, target_width, include_filename, expose_images, expose_images_persist, frigate_retry_attempts, frigate_retry_seconds):
         """Wrapper for client.add_frame for videos"""
         tmp_clips_dir = f"/config/custom_components/{DOMAIN}/tmp_clips"
         tmp_frames_dir = f"/config/custom_components/{DOMAIN}/tmp_frames"
@@ -307,8 +307,8 @@ async def add_videos(self, video_paths, event_ids, max_frames, target_width, inc
                 try:
                     base_url = get_url(self.hass)
                     frigate_url = base_url + "/api/frigate/notifications/" + event_id + "/clip.mp4"
-                    clip_data = await self.client._fetch(frigate_url)
-
+                    clip_data = await self.client._fetch(frigate_url, max_retries=frigate_retry_attempts, retry_delay=frigate_retry_seconds)
+                    
                     if not clip_data:
                         raise ServiceValidationError(
                             f"Failed to fetch frigate clip {event_id}")
@@ -351,8 +351,9 @@ async def add_videos(self, video_paths, event_ids, max_frames, target_width, inc
                         ffmpeg_cmd = [
                             "ffmpeg",
                             "-i", video_path,
-                            "-vf", f"fps=1/{interval},select='eq(n\\,0)+not(mod(n\\,{interval}))'", os.path.join(
-                                tmp_frames_dir, "frame%04d.jpg")
+                            "-vf", f"fps=fps='source_fps',select='eq(n\\,0)+not(mod(n\\,{interval}))'", 
+                            "-fps_mode", "passthrough",
+                            os.path.join(tmp_frames_dir, "frame%04d.jpg")
                         ]
                         # Run ffmpeg command
                         await self.hass.loop.run_in_executor(None, os.system, " ".join(ffmpeg_cmd))

custom_components/llmvision/services.yaml

@@ -160,6 +160,30 @@ video_analyzer:
       selector:
         text:
           multiline: true
+    frigate_retry_attempts:
+      name: Frigate Retry Attempts
+      description: How many times to retry fetching the video clip from Frigate.  Clips are not always available from Frigate as soon as the event has ended.  
+          Slower machines or longer clips may need additional attempts.  Increase this if you see errors fetching the clips from Frigate in your automation traces.
+      required: false
+      example: 2
+      default: 2
+      selector:
+        number:
+          min: 1
+          max: 10
+          step: 1
+    frigate_retry_seconds:
+      name: Frigate Retry Seconds
+      description: How long to wait between retries to fetch the video clip from Frigate.  Clips are not always available from Frigate as soon as the event has ended.  
+          Slower machines or longer clips may need additional attempts.  Increase this if you see errors fetching the clips from Frigate in your automation traces.
+      required: false
+      example: 1
+      default: 1
+      selector:
+        number:
+          min: 1
+          max: 10
+          step: 1
     max_frames:
       name: Max Frames
       description: How many frames to analyze. Picks frames with the most movement.