From 3ef0e19494889daef65ceec8036131524511daaa Mon Sep 17 00:00:00 2001 From: valentinfrlch Date: Mon, 21 Oct 2024 21:30:14 +0200 Subject: [PATCH] execution time is now accounted for in recording timing --- blueprints/event_summary.yaml | 12 +++--- custom_components/llmvision/media_handlers.py | 42 +++++++++++++------ .../llmvision/request_handlers.py | 7 ++-- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/blueprints/event_summary.yaml b/blueprints/event_summary.yaml index 7eeafb1..cf02d7f 100644 --- a/blueprints/event_summary.yaml +++ b/blueprints/event_summary.yaml @@ -213,14 +213,14 @@ action: sequence: - service: llmvision.image_analyzer data: - image_entity: "{{trigger.payload_json['after']['camera']}}" + image_entity: "{{ ['camera.' + trigger.payload_json['after']['camera']|lower] }}" provider: !input provider model: !input model message: "{{importance_prompt}}" include_filename: true target_width: 1280 detail: low - max_tokens: 5 + max_tokens: 1 temperature: 0.1 response_variable: importance - conditions: @@ -236,7 +236,7 @@ action: include_filename: true target_width: 1280 detail: low - max_tokens: 5 + max_tokens: 1 temperature: 0.1 response_variable: importance @@ -264,7 +264,7 @@ action: data: video: "{{video}}" image: "{{image}}" - entity_id: "{{trigger.entity_id if mode=='Camera' else ''}}" + entity_id: "{{trigger.entity_id | default('') if mode=='Camera' else ''}}" url: !input tap_navigate #iOS clickAction: !input tap_navigate #Android tag: "{{tag}}" @@ -278,7 +278,7 @@ action: sequence: - service: llmvision.video_analyzer data: - event_id: '{{trigger.payload_json["after"]["id"]}}' + event_id: "{{ trigger.payload_json['after']['id'] }}" provider: !input provider model: !input model message: !input message @@ -319,7 +319,7 @@ action: group: "{{group}}" image: "{{image}}" video: "{{video}}" - entity_id: "{{trigger.entity_id if mode=='Camera' else ''}}" + entity_id: "{{trigger.entity_id | default('') if mode=='Camera' else ''}}" url: !input tap_navigate #iOS clickAction: !input tap_navigate #Android push: diff --git a/custom_components/llmvision/media_handlers.py b/custom_components/llmvision/media_handlers.py index fcad93b..4476de0 100644 --- a/custom_components/llmvision/media_handlers.py +++ b/custom_components/llmvision/media_handlers.py @@ -95,7 +95,6 @@ async def resize_image(self, target_width, image_path=None, image_data=None, img img_byte_arr.write(image_data) img = await self.hass.loop.run_in_executor(None, Image.open, img_byte_arr) with img: - _LOGGER.debug(f"Image format: {img.format}") img = self._convert_to_rgb(img) # calculate new height based on aspect ratio width, height = img.size @@ -133,7 +132,6 @@ async def record(self, image_entities, duration, max_frames, target_width, inclu import asyncio interval = 1 if duration < 3 else 2 if duration < 10 else 4 if duration < 30 else 6 if duration < 60 else 10 - duration += 1 # add 1 second in case the first fetch fails camera_frames = {} # Record on a separate thread for each camera @@ -142,18 +140,24 @@ async def record_camera(image_entity, camera_number): frame_counter = 0 frames = {} previous_frame = None - while time.time() - start < duration: + iteration_time = 0 + + while time.time() - start < duration + iteration_time: + fetch_start_time = time.time() base_url = get_url(self.hass) - # fetched every cycle to ensure latest access token frame_url = base_url + \ self.hass.states.get(image_entity).attributes.get( 'entity_picture') frame_data = await self.client._fetch(frame_url) - - # Skip frame when fetch failed + + # Skip frame if fetch failed if not frame_data: continue + fetch_duration = time.time() - fetch_start_time + _LOGGER.info(f"Fetched {image_entity} in {fetch_duration:.2f} seconds") + + preprocessing_start_time = time.time() img = Image.open(io.BytesIO(frame_data)) current_frame_gray = np.array(img.convert('L')) @@ -178,7 +182,17 @@ async def record_camera(image_entity, camera_number): # Initialize previous_frame with the first frame previous_frame = current_frame_gray - await asyncio.sleep(interval) + preprocessing_duration = time.time() - preprocessing_start_time + _LOGGER.info(f"Preprocessing took: {preprocessing_duration:.2f} seconds") + + adjusted_interval = max(0, interval - fetch_duration - preprocessing_duration) + + if iteration_time == 0: + iteration_time = time.time() - start + _LOGGER.info(f"First iteration took: {iteration_time:.2f} seconds, interval adjusted to: {adjusted_interval}") + + await asyncio.sleep(adjusted_interval) + camera_frames.update({image_entity: frames}) _LOGGER.info(f"Recording {', '.join([entity.replace( @@ -217,10 +231,12 @@ async def add_images(self, image_entities, image_paths, target_width, include_fi self.hass.states.get(image_entity).attributes.get( 'entity_picture') image_data = await self.client._fetch(image_url) - - # Skip frame when fetch failed + + # Skip frame if fetch failed if not image_data: - continue + if len(image_entities) == 1: + raise ServiceValidationError( + f"Failed to fetch image from {image_entity}") # If entity snapshot requested, use entity name as 'filename' self.client.add_frame( @@ -265,10 +281,10 @@ async def add_videos(self, video_paths, event_ids, max_frames, target_width, inc base_url = get_url(self.hass) frigate_url = base_url + "/api/frigate/notifications/" + event_id + "/clip.mp4" clip_data = await self.client._fetch(frigate_url) - - # Skip frame when fetch failed + if not clip_data: - continue + raise ServiceValidationError( + f"Failed to fetch frigate clip {event_id}") # create tmp dir to store video clips os.makedirs(tmp_clips_dir, exist_ok=True) diff --git a/custom_components/llmvision/request_handlers.py b/custom_components/llmvision/request_handlers.py index b4efdbd..74f0660 100644 --- a/custom_components/llmvision/request_handlers.py +++ b/custom_components/llmvision/request_handlers.py @@ -1,6 +1,7 @@ from homeassistant.exceptions import ServiceValidationError from homeassistant.helpers.aiohttp_client import async_get_clientsession import logging +import asyncio import inspect from .const import ( DOMAIN, @@ -415,8 +416,7 @@ async def _post(self, url, headers, data): _LOGGER.info(f"Response data: {response_data}") return response_data - async def _fetch(self, url, max_retries=1, retry_delay=1): - import asyncio + async def _fetch(self, url, max_retries=2, retry_delay=1): """Fetch image from url and return image data""" retries = 0 while retries < max_retries: @@ -435,8 +435,7 @@ async def _fetch(self, url, max_retries=1, retry_delay=1): _LOGGER.error(f"Fetch failed: {e}") retries += 1 await asyncio.sleep(retry_delay) - - _LOGGER.error(f"Failed to fetch {url} after {max_retries} retries") + _LOGGER.warning(f"Failed to fetch {url} after {max_retries} retries") return None def _validate_call(self, provider, api_key, base64_images, ip_address=None, port=None):