From 0db6c45bf9fd34c9f22caf0491fb36a040e3ee33 Mon Sep 17 00:00:00 2001 From: Ryan Steckler Date: Mon, 9 Dec 2024 20:00:58 -0800 Subject: [PATCH 1/9] Configurable retries on fetching Frigate clips --- custom_components/llmvision/__init__.py | 8 ++++++- custom_components/llmvision/const.py | 2 ++ custom_components/llmvision/media_handlers.py | 6 ++--- custom_components/llmvision/services.yaml | 24 +++++++++++++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/custom_components/llmvision/__init__.py b/custom_components/llmvision/__init__.py index 3d94311..5fb9c01 100644 --- a/custom_components/llmvision/__init__.py +++ b/custom_components/llmvision/__init__.py @@ -24,6 +24,8 @@ IMAGE_ENTITY, VIDEO_FILE, EVENT_ID, + FRIGATE_RETRY_ATTEMPTS, + FRIGATE_RETRY_SECONDS, INTERVAL, DURATION, MAX_FRAMES, @@ -227,6 +229,8 @@ def __init__(self, data_call): "\n") if data_call.data.get(EVENT_ID) else None self.interval = int(data_call.data.get(INTERVAL, 2)) self.duration = int(data_call.data.get(DURATION, 10)) + self.frigate_retry_attempts = int(data_call.data.get(FRIGATE_RETRY_ATTEMPTS, 2)) + self.frigate_retry_seconds = int(data_call.data.get(FRIGATE_RETRY_SECONDS, 1)) self.max_frames = int(data_call.data.get(MAX_FRAMES, 3)) self.target_width = data_call.data.get(TARGET_WIDTH, 3840) self.temperature = float(data_call.data.get(TEMPERATURE, 0.3)) @@ -285,7 +289,9 @@ async def video_analyzer(data_call): max_frames=call.max_frames, target_width=call.target_width, include_filename=call.include_filename, - expose_images=call.expose_images + expose_images=call.expose_images, + frigate_retry_attempts=call.frigate_retry_attempts, + frigate_retry_seconds=call.frigate_retry_seconds ) response = await client.make_request(call) await _remember(hass, call, start, response) diff --git a/custom_components/llmvision/const.py b/custom_components/llmvision/const.py index 4bbd7f8..b554ff9 100644 --- a/custom_components/llmvision/const.py +++ b/custom_components/llmvision/const.py @@ -31,6 +31,8 @@ EVENT_ID = 'event_id' INTERVAL = 'interval' DURATION = 'duration' +FRIGATE_RETRY_ATTEMPTS = 'frigate_retry_attempts' +FRIGATE_RETRY_SECONDS = 'frigate_retry_seconds' MAX_FRAMES = 'max_frames' DETAIL = 'detail' TEMPERATURE = 'temperature' diff --git a/custom_components/llmvision/media_handlers.py b/custom_components/llmvision/media_handlers.py index f37ab8c..69a03ab 100644 --- a/custom_components/llmvision/media_handlers.py +++ b/custom_components/llmvision/media_handlers.py @@ -294,7 +294,7 @@ async def add_images(self, image_entities, image_paths, target_width, include_fi raise ServiceValidationError(f"Error: {e}") return self.client - async def add_videos(self, video_paths, event_ids, max_frames, target_width, include_filename, expose_images): + async def add_videos(self, video_paths, event_ids, max_frames, target_width, include_filename, expose_images, frigate_retry_attempts, frigate_retry_seconds): """Wrapper for client.add_frame for videos""" tmp_clips_dir = f"/config/custom_components/{DOMAIN}/tmp_clips" tmp_frames_dir = f"/config/custom_components/{DOMAIN}/tmp_frames" @@ -306,8 +306,8 @@ async def add_videos(self, video_paths, event_ids, max_frames, target_width, inc try: base_url = get_url(self.hass) frigate_url = base_url + "/api/frigate/notifications/" + event_id + "/clip.mp4" - clip_data = await self.client._fetch(frigate_url) - + clip_data = await self.client._fetch(frigate_url, max_retries=frigate_retry_attempts, retry_delay=frigate_retry_seconds) + if not clip_data: raise ServiceValidationError( f"Failed to fetch frigate clip {event_id}") diff --git a/custom_components/llmvision/services.yaml b/custom_components/llmvision/services.yaml index 78ad818..d40bfec 100644 --- a/custom_components/llmvision/services.yaml +++ b/custom_components/llmvision/services.yaml @@ -160,6 +160,30 @@ video_analyzer: selector: text: multiline: true + frigate_retry_attempts: + name: Frigate Retry Attempts + description: How many times to retry fetching the video clip from Frigate. Clips are not always available from Frigate as soon as the event has ended. + Slower machines or longer clips may need additional attempts. Increase this if you see errors fetching the clips from Frigate in your automation traces. + required: false + example: 2 + default: 2 + selector: + number: + min: 1 + max: 10 + step: 1 + frigate_retry_seconds: + name: Frigate Retry Seconds + description: How long to wait between retries to fetch the video clip from Frigate. Clips are not always available from Frigate as soon as the event has ended. + Slower machines or longer clips may need additional attempts. Increase this if you see errors fetching the clips from Frigate in your automation traces. + required: false + example: 1 + default: 1 + selector: + number: + min: 1 + max: 10 + step: 1 max_frames: name: Max Frames description: How many frames to analyze. Picks frames with the most movement. From 0f4d94fbbc7ad0f63d98491e024390a5aaf4cd0a Mon Sep 17 00:00:00 2001 From: Ryan Steckler Date: Mon, 9 Dec 2024 21:48:08 -0800 Subject: [PATCH 2/9] fixed ffmpeg command --- custom_components/llmvision/media_handlers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/custom_components/llmvision/media_handlers.py b/custom_components/llmvision/media_handlers.py index f37ab8c..9cab83f 100644 --- a/custom_components/llmvision/media_handlers.py +++ b/custom_components/llmvision/media_handlers.py @@ -347,8 +347,9 @@ async def add_videos(self, video_paths, event_ids, max_frames, target_width, inc ffmpeg_cmd = [ "ffmpeg", "-i", video_path, - "-vf", f"fps=1/{interval},select='eq(n\\,0)+not(mod(n\\,{interval}))'", os.path.join( - tmp_frames_dir, "frame%04d.jpg") + "-vf", f"fps=fps='source_fps',select='eq(n\\,0)+not(mod(n\\,{interval}))'", + "-fps_mode", "passthrough", + os.path.join(tmp_frames_dir, "frame%04d.jpg") ] # Run ffmpeg command await self.hass.loop.run_in_executor(None, os.system, " ".join(ffmpeg_cmd)) From 297c453a0f2bc317aa196da4bee8a6e7174019d2 Mon Sep 17 00:00:00 2001 From: Michael Rappazzo Date: Tue, 10 Dec 2024 13:06:34 -0500 Subject: [PATCH 3/9] event_summary blueprint: adjust the device name sanitization The device name sanitize code was changed from "remove all non-conforming characters and then convert some special ones to '_'" to "convert some special characters to '_' and then remove all other non-conforming characters. In the new version, both of these operations were changed to use regex replace only. Note that in the "special characters" expression `[' -]` the '-' does not need to be escaped because it is the last item in the regex character class. The "non-conforming characters" regex was changed to `[^a-z0-9_]`, which reflects that the special characters no longer need to be included. An important addition to the "special characters" is the single-quote, which was previously omitted. --- blueprints/event_summary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index d204a21..ac4915e 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -170,7 +170,7 @@ variables: {% set ns = namespace(device_names=[]) %} {% for device_id in notify_devices %} {% set device_name = device_attr(device_id, "name") %} - {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[^a-z0-9_\- ]", "") | replace(" ", "_") | replace("-", "_") %} + {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[' -]", "_") | regex_replace("[^a-z0-9_]", "") %} {% set ns.device_names = ns.device_names + [sanitized_name] %} {% endfor %} {{ ns.device_names }} From 476820c54e8a1f8ab8358c8d5d7c22bf06d9e6b4 Mon Sep 17 00:00:00 2001 From: Michael Rappazzo Date: Mon, 16 Dec 2024 15:19:38 -0500 Subject: [PATCH 4/9] blueprint: re-implement cooldown with singlemode execution and a delay Force the automation to use single mode with suppressed warnings (see https://www.home-assistant.io/docs/automation/modes/). To implement the cooldown, add a delay as the last action in the actions. Also remove the cooldown from the automation conditions. The main difference here is that no new triggers will arrive while the automation is running. The condition is smaller now, and is limited to the content of the automation, and not managing how the automation is run. With this narrowing, it opens up the possibility of adding other conditions to the blueprint config in the future. --- blueprints/event_summary.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index ac4915e..faabe94 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -230,6 +230,10 @@ variables: Use "critical" only for possible burglaries and similar events. "time-sensitive" could be a courier at the front door or an event of similar importance. Reply with these replies exactly. +max_exceeded: silent + +mode: single + trigger: - platform: mqtt topic: "frigate/events" @@ -247,9 +251,7 @@ condition: - condition: template value_template: > {% if mode == 'Frigate' %} - {{ trigger.payload_json["type"] == "end" and (state_attr(this.entity_id, 'last_triggered') is none or (now() - state_attr(this.entity_id, 'last_triggered')).total_seconds() / 60 > cooldown) and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list }} - {% else %} - {{ state_attr(this.entity_id, 'last_triggered') is none or (now() - state_attr(this.entity_id, 'last_triggered')).total_seconds() / 60 > cooldown }} + {{ trigger.payload_json["type"] == "end" and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list }} {% endif %} @@ -389,3 +391,5 @@ action: tag: "{{tag}}" group: "{{group}}" interruption-level: passive + +- delay: 00:{{cooldown|int}}:00 From 1f14fea14ddc7a34a23abb52130677bd6f3c562e Mon Sep 17 00:00:00 2001 From: Michael Rappazzo Date: Sat, 21 Dec 2024 09:29:23 -0500 Subject: [PATCH 5/9] blueprint: reformat the variable descriptions Include a newline in the description for variables which describe their applicability for frigate mode, camera mode, or both. Also, add a suggestion for the 'tap naviagation' to link directly to the input video or image. --- blueprints/event_summary.yaml | 42 ++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index ac4915e..35d7088 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -2,7 +2,7 @@ blueprint: name: AI Event Summary (LLM Vision v1.3.1) author: valentinfrlch description: > - AI-powered security event summaries for frigate or camera entities. + AI-powered security event summaries for frigate or camera entities. Sends a notification with a preview to your phone that is updated dynamically when the AI summary is available. domain: automation source_url: https://github.com/valentinfrlch/ha-llmvision/blob/main/blueprints/event_summary.yaml @@ -42,7 +42,10 @@ blueprint: integration: mobile_app camera_entities: name: Camera Entities - description: (Camera and Frigate mode) List of camera entities to monitor + description: >- + (Camera and Frigate mode) + + List of camera entities to monitor default: [] selector: entity: @@ -51,14 +54,20 @@ blueprint: domain: camera trigger_state: name: Trigger State - description: (Camera mode only) Trigger the automation when your cameras change to this state. + description: >- + (Camera mode only) + + Trigger the automation when your cameras change to this state. default: 'recording' selector: text: multiline: false motion_sensors: name: Motion Sensor - description: (Camera mode only) Set if your cameras don't change state. Use the same order used for camera entities. + description: >- + (Camera mode only) + + Set if your cameras don't change state. Use the same order used for camera entities. default: [] selector: entity: @@ -67,7 +76,10 @@ blueprint: domain: binary_sensor preview_mode: name: Preview Mode - description: (Camera mode only) Choose between a live preview or a snapshot of the event + description: >- + (Camera mode only) + + Choose between a live preview or a snapshot of the event default: 'Live Preview' selector: select: @@ -84,14 +96,21 @@ blueprint: max: 60 tap_navigate: name: Tap Navigate - description: Path to navigate to when notification is opened (e.g. /lovelace/cameras) + description: >- + Path to navigate to when notification is opened (e.g. /lovelace/cameras). + + To have use the same input which was sent to the ai engine, use + `{{video if video != '''' else image}}` default: "/lovelace/0" selector: text: multiline: false duration: name: Duration - description: (Camera mode only) How long to record before analyzing (in seconds) + description: >- + (Camera mode only) + + How long to record before analyzing (in seconds) default: 5 selector: number: @@ -99,7 +118,10 @@ blueprint: max: 60 max_frames: name: Max Frames - description: (Camera and Frigate mode) How many frames to analyze. Picks frames with the most movement. + description: >- + (Camera and Frigate mode) + + How many frames to analyze. Picks frames with the most movement. default: 3 selector: number: @@ -293,7 +315,7 @@ action: max_tokens: 3 temperature: 0.1 response_variable: importance - + # Cancel automation if event not deemed important - choose: - conditions: @@ -365,7 +387,7 @@ action: temperature: !input temperature expose_images: "{{true if preview_mode == 'Snapshot'}}" response_variable: response - + - choose: - conditions: From 6d754f2a27a68a3842e3d92dd91ed106c28da335 Mon Sep 17 00:00:00 2001 From: Michael Rappazzo Date: Sat, 21 Dec 2024 08:12:30 -0500 Subject: [PATCH 6/9] blueprint: add a filter for frigate object detection The frigate payload includes a label which indicates what the detected object is. This can be useful to separate llm-vision automations to react differently for different objects. For example, if a bird is detected, the ai instructions could include something like, "if you see a bird, try to identify what species it is". This discription doesn't make much sense if the detected object is a person, and including separate instructions to the ai makes it more confusing on both the human and ai side of things. Allowing for separate instructions by object will make for easier to understand automations. --- blueprints/event_summary.yaml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index faabe94..128fa9f 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -49,6 +49,17 @@ blueprint: multiple: true filter: domain: camera + object_type: + name: Included Object Type(s) + description: >- + (Frigate mode only) + + Only run if frigate labels the object as one of these. (person, dog, bird, etc) + default: [] + selector: + text: + multiline: false + multiple: true trigger_state: name: Trigger State description: (Camera mode only) Trigger the automation when your cameras change to this state. @@ -175,6 +186,7 @@ variables: {% endfor %} {{ ns.device_names }} camera_entities_list: !input camera_entities + object_types_list: !input object_type motion_sensors_list: !input motion_sensors camera_entity: > {% if mode == 'Camera' %} @@ -251,7 +263,10 @@ condition: - condition: template value_template: > {% if mode == 'Frigate' %} - {{ trigger.payload_json["type"] == "end" and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list }} + {{ trigger.payload_json["type"] == "end" + and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list + and ((object_types_list|length) == 0 or ((trigger.payload_json['after']['label']|lower) in object_types_list)) + }} {% endif %} From 25084c14b13953438480a59cf79c905d8d342f35 Mon Sep 17 00:00:00 2001 From: Moz <97829410+SleepyMoz@users.noreply.github.com> Date: Mon, 23 Dec 2024 06:54:26 +0000 Subject: [PATCH 7/9] Fixed a bug with importing to HA --- blueprints/event_summary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index e2e9b9c..380bccd 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -429,4 +429,4 @@ action: group: "{{group}}" interruption-level: passive -- delay: 00:{{cooldown|int}}:00 +- delay: '00:{{cooldown|int}}:00' From 2299060e049b9772324015fe0d097a3ae292a3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Fr=C3=B6hlich?= <85313672+valentinfrlch@users.noreply.github.com> Date: Mon, 23 Dec 2024 08:33:10 +0100 Subject: [PATCH 8/9] Update event_summary.yaml Added two spaces in front of "- delay" --- blueprints/event_summary.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index 380bccd..b422e13 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -429,4 +429,4 @@ action: group: "{{group}}" interruption-level: passive -- delay: '00:{{cooldown|int}}:00' + - delay: '00:{{cooldown|int}}:00' From d7335e04a3da96e11cab61a1aaf45a9dbc4f47e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Fr=C3=B6hlich?= <85313672+valentinfrlch@users.noreply.github.com> Date: Mon, 23 Dec 2024 14:51:57 +0100 Subject: [PATCH 9/9] (Blueprint) Camera mode hotfix Fixes #141 --- blueprints/event_summary.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index b422e13..ea5304e 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -289,6 +289,8 @@ condition: and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list and ((object_types_list|length) == 0 or ((trigger.payload_json['after']['label']|lower) in object_types_list)) }} + {%else%} + true {% endif %}