diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index d204a21..ea5304e 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -2,7 +2,7 @@ blueprint: name: AI Event Summary (LLM Vision v1.3.1) author: valentinfrlch description: > - AI-powered security event summaries for frigate or camera entities. + AI-powered security event summaries for frigate or camera entities. Sends a notification with a preview to your phone that is updated dynamically when the AI summary is available. domain: automation source_url: https://github.com/valentinfrlch/ha-llmvision/blob/main/blueprints/event_summary.yaml @@ -42,23 +42,43 @@ blueprint: integration: mobile_app camera_entities: name: Camera Entities - description: (Camera and Frigate mode) List of camera entities to monitor + description: >- + (Camera and Frigate mode) + + List of camera entities to monitor default: [] selector: entity: multiple: true filter: domain: camera + object_type: + name: Included Object Type(s) + description: >- + (Frigate mode only) + + Only run if frigate labels the object as one of these. (person, dog, bird, etc) + default: [] + selector: + text: + multiline: false + multiple: true trigger_state: name: Trigger State - description: (Camera mode only) Trigger the automation when your cameras change to this state. + description: >- + (Camera mode only) + + Trigger the automation when your cameras change to this state. default: 'recording' selector: text: multiline: false motion_sensors: name: Motion Sensor - description: (Camera mode only) Set if your cameras don't change state. Use the same order used for camera entities. + description: >- + (Camera mode only) + + Set if your cameras don't change state. Use the same order used for camera entities. default: [] selector: entity: @@ -67,7 +87,10 @@ blueprint: domain: binary_sensor preview_mode: name: Preview Mode - description: (Camera mode only) Choose between a live preview or a snapshot of the event + description: >- + (Camera mode only) + + Choose between a live preview or a snapshot of the event default: 'Live Preview' selector: select: @@ -84,14 +107,21 @@ blueprint: max: 60 tap_navigate: name: Tap Navigate - description: Path to navigate to when notification is opened (e.g. /lovelace/cameras) + description: >- + Path to navigate to when notification is opened (e.g. /lovelace/cameras). + + To have use the same input which was sent to the ai engine, use + `{{video if video != '''' else image}}` default: "/lovelace/0" selector: text: multiline: false duration: name: Duration - description: (Camera mode only) How long to record before analyzing (in seconds) + description: >- + (Camera mode only) + + How long to record before analyzing (in seconds) default: 5 selector: number: @@ -99,7 +129,10 @@ blueprint: max: 60 max_frames: name: Max Frames - description: (Camera and Frigate mode) How many frames to analyze. Picks frames with the most movement. + description: >- + (Camera and Frigate mode) + + How many frames to analyze. Picks frames with the most movement. default: 3 selector: number: @@ -170,11 +203,12 @@ variables: {% set ns = namespace(device_names=[]) %} {% for device_id in notify_devices %} {% set device_name = device_attr(device_id, "name") %} - {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[^a-z0-9_\- ]", "") | replace(" ", "_") | replace("-", "_") %} + {% set sanitized_name = "mobile_app_" + device_name | lower | regex_replace("[' -]", "_") | regex_replace("[^a-z0-9_]", "") %} {% set ns.device_names = ns.device_names + [sanitized_name] %} {% endfor %} {{ ns.device_names }} camera_entities_list: !input camera_entities + object_types_list: !input object_type motion_sensors_list: !input motion_sensors camera_entity: > {% if mode == 'Camera' %} @@ -230,6 +264,10 @@ variables: Use "critical" only for possible burglaries and similar events. "time-sensitive" could be a courier at the front door or an event of similar importance. Reply with these replies exactly. +max_exceeded: silent + +mode: single + trigger: - platform: mqtt topic: "frigate/events" @@ -247,9 +285,12 @@ condition: - condition: template value_template: > {% if mode == 'Frigate' %} - {{ trigger.payload_json["type"] == "end" and (state_attr(this.entity_id, 'last_triggered') is none or (now() - state_attr(this.entity_id, 'last_triggered')).total_seconds() / 60 > cooldown) and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list }} - {% else %} - {{ state_attr(this.entity_id, 'last_triggered') is none or (now() - state_attr(this.entity_id, 'last_triggered')).total_seconds() / 60 > cooldown }} + {{ trigger.payload_json["type"] == "end" + and ('camera.' + trigger.payload_json['after']['camera']|lower) in camera_entities_list + and ((object_types_list|length) == 0 or ((trigger.payload_json['after']['label']|lower) in object_types_list)) + }} + {%else%} + true {% endif %} @@ -293,7 +334,7 @@ action: max_tokens: 3 temperature: 0.1 response_variable: importance - + # Cancel automation if event not deemed important - choose: - conditions: @@ -365,7 +406,7 @@ action: temperature: !input temperature expose_images: "{{true if preview_mode == 'Snapshot'}}" response_variable: response - + - choose: - conditions: @@ -389,3 +430,5 @@ action: tag: "{{tag}}" group: "{{group}}" interruption-level: passive + + - delay: '00:{{cooldown|int}}:00' diff --git a/custom_components/llmvision/__init__.py b/custom_components/llmvision/__init__.py index 75ddafc..ef72029 100644 --- a/custom_components/llmvision/__init__.py +++ b/custom_components/llmvision/__init__.py @@ -24,6 +24,8 @@ IMAGE_ENTITY, VIDEO_FILE, EVENT_ID, + FRIGATE_RETRY_ATTEMPTS, + FRIGATE_RETRY_SECONDS, INTERVAL, DURATION, MAX_FRAMES, @@ -228,6 +230,8 @@ def __init__(self, data_call): "\n") if data_call.data.get(EVENT_ID) else None self.interval = int(data_call.data.get(INTERVAL, 2)) self.duration = int(data_call.data.get(DURATION, 10)) + self.frigate_retry_attempts = int(data_call.data.get(FRIGATE_RETRY_ATTEMPTS, 2)) + self.frigate_retry_seconds = int(data_call.data.get(FRIGATE_RETRY_SECONDS, 1)) self.max_frames = int(data_call.data.get(MAX_FRAMES, 3)) self.target_width = data_call.data.get(TARGET_WIDTH, 3840) self.temperature = float(data_call.data.get(TEMPERATURE, 0.3)) @@ -288,7 +292,9 @@ async def video_analyzer(data_call): target_width=call.target_width, include_filename=call.include_filename, expose_images=call.expose_images, - expose_images_persist=call.expose_images_persist + expose_images_persist=call.expose_images_persist, + frigate_retry_attempts=call.frigate_retry_attempts, + frigate_retry_seconds=call.frigate_retry_seconds ) response = await client.make_request(call) await _remember(hass, call, start, response) diff --git a/custom_components/llmvision/const.py b/custom_components/llmvision/const.py index 3856516..6e0a61f 100644 --- a/custom_components/llmvision/const.py +++ b/custom_components/llmvision/const.py @@ -31,6 +31,8 @@ EVENT_ID = 'event_id' INTERVAL = 'interval' DURATION = 'duration' +FRIGATE_RETRY_ATTEMPTS = 'frigate_retry_attempts' +FRIGATE_RETRY_SECONDS = 'frigate_retry_seconds' MAX_FRAMES = 'max_frames' DETAIL = 'detail' TEMPERATURE = 'temperature' diff --git a/custom_components/llmvision/media_handlers.py b/custom_components/llmvision/media_handlers.py index 7436fcb..f58f436 100644 --- a/custom_components/llmvision/media_handlers.py +++ b/custom_components/llmvision/media_handlers.py @@ -294,7 +294,7 @@ async def add_images(self, image_entities, image_paths, target_width, include_fi raise ServiceValidationError(f"Error: {e}") return self.client - async def add_videos(self, video_paths, event_ids, max_frames, target_width, include_filename, expose_images, expose_images_persist): + async def add_videos(self, video_paths, event_ids, max_frames, target_width, include_filename, expose_images, expose_images_persist, frigate_retry_attempts, frigate_retry_seconds): """Wrapper for client.add_frame for videos""" tmp_clips_dir = f"/config/custom_components/{DOMAIN}/tmp_clips" tmp_frames_dir = f"/config/custom_components/{DOMAIN}/tmp_frames" @@ -307,8 +307,8 @@ async def add_videos(self, video_paths, event_ids, max_frames, target_width, inc try: base_url = get_url(self.hass) frigate_url = base_url + "/api/frigate/notifications/" + event_id + "/clip.mp4" - clip_data = await self.client._fetch(frigate_url) - + clip_data = await self.client._fetch(frigate_url, max_retries=frigate_retry_attempts, retry_delay=frigate_retry_seconds) + if not clip_data: raise ServiceValidationError( f"Failed to fetch frigate clip {event_id}") @@ -351,8 +351,9 @@ async def add_videos(self, video_paths, event_ids, max_frames, target_width, inc ffmpeg_cmd = [ "ffmpeg", "-i", video_path, - "-vf", f"fps=1/{interval},select='eq(n\\,0)+not(mod(n\\,{interval}))'", os.path.join( - tmp_frames_dir, "frame%04d.jpg") + "-vf", f"fps=fps='source_fps',select='eq(n\\,0)+not(mod(n\\,{interval}))'", + "-fps_mode", "passthrough", + os.path.join(tmp_frames_dir, "frame%04d.jpg") ] # Run ffmpeg command await self.hass.loop.run_in_executor(None, os.system, " ".join(ffmpeg_cmd)) diff --git a/custom_components/llmvision/services.yaml b/custom_components/llmvision/services.yaml index 6a337a6..cbd5339 100644 --- a/custom_components/llmvision/services.yaml +++ b/custom_components/llmvision/services.yaml @@ -160,6 +160,30 @@ video_analyzer: selector: text: multiline: true + frigate_retry_attempts: + name: Frigate Retry Attempts + description: How many times to retry fetching the video clip from Frigate. Clips are not always available from Frigate as soon as the event has ended. + Slower machines or longer clips may need additional attempts. Increase this if you see errors fetching the clips from Frigate in your automation traces. + required: false + example: 2 + default: 2 + selector: + number: + min: 1 + max: 10 + step: 1 + frigate_retry_seconds: + name: Frigate Retry Seconds + description: How long to wait between retries to fetch the video clip from Frigate. Clips are not always available from Frigate as soon as the event has ended. + Slower machines or longer clips may need additional attempts. Increase this if you see errors fetching the clips from Frigate in your automation traces. + required: false + example: 1 + default: 1 + selector: + number: + min: 1 + max: 10 + step: 1 max_frames: name: Max Frames description: How many frames to analyze. Picks frames with the most movement.