From 83f7a7cc1a2f795dd04c2b65f7a5443a2a10b1ef Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 22 Jul 2024 13:30:22 -0500 Subject: [PATCH 01/10] Bump aiohttp to 3.10.0b1 changelog: https://github.com/aio-libs/aiohttp/compare/v3.9.5...v3.10.0b1 --- homeassistant/package_constraints.txt | 2 +- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 3eaa0b06619e0d..6cdf888e4e6609 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -5,7 +5,7 @@ aiodiscover==2.1.0 aiodns==3.2.0 aiohttp-fast-url-dispatcher==0.3.0 aiohttp-fast-zlib==0.1.1 -aiohttp==3.9.5 +aiohttp==3.10.0b1 aiohttp_cors==0.7.0 aiozoneinfo==0.2.1 astral==2.2 diff --git a/pyproject.toml b/pyproject.toml index 113fef6bfbc9ad..6231539f7bc1f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ requires-python = ">=3.12.0" dependencies = [ "aiodns==3.2.0", - "aiohttp==3.9.5", + "aiohttp==3.10.0b1", "aiohttp_cors==0.7.0", "aiohttp-fast-url-dispatcher==0.3.0", "aiohttp-fast-zlib==0.1.1", diff --git a/requirements.txt b/requirements.txt index a729f09472bc6a..674b31af7b422a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # Home Assistant Core aiodns==3.2.0 -aiohttp==3.9.5 +aiohttp==3.10.0b1 aiohttp_cors==0.7.0 aiohttp-fast-url-dispatcher==0.3.0 aiohttp-fast-zlib==0.1.1 From 15877a9566b05ff24a74085a054fe809f26f57ea Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 22 Jul 2024 13:37:46 -0500 Subject: [PATCH 02/10] drop aiohttp-fast-url-dispatcher --- homeassistant/components/http/__init__.py | 5 ----- homeassistant/package_constraints.txt | 1 - pyproject.toml | 1 - requirements.txt | 1 - script/licenses.py | 1 - 5 files changed, 9 deletions(-) diff --git a/homeassistant/components/http/__init__.py b/homeassistant/components/http/__init__.py index 0d86ab57d3f982..5b68f91e494807 100644 --- a/homeassistant/components/http/__init__.py +++ b/homeassistant/components/http/__init__.py @@ -22,7 +22,6 @@ from aiohttp.typedefs import JSONDecoder, StrOrURL from aiohttp.web_exceptions import HTTPMovedPermanently, HTTPRedirection from aiohttp.web_protocol import RequestHandler -from aiohttp_fast_url_dispatcher import FastUrlDispatcher, attach_fast_url_dispatcher from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa @@ -335,10 +334,6 @@ def __init__( "max_field_size": MAX_LINE_SIZE, }, ) - # By default aiohttp does a linear search for routing rules, - # we have a lot of routes, so use a dict lookup with a fallback - # to the linear search. - attach_fast_url_dispatcher(self.app, FastUrlDispatcher()) self.hass = hass self.ssl_certificate = ssl_certificate self.ssl_peer_certificate = ssl_peer_certificate diff --git a/homeassistant/package_constraints.txt b/homeassistant/package_constraints.txt index 6cdf888e4e6609..a7435e844f5fca 100644 --- a/homeassistant/package_constraints.txt +++ b/homeassistant/package_constraints.txt @@ -3,7 +3,6 @@ aiodhcpwatcher==1.0.2 aiodiscover==2.1.0 aiodns==3.2.0 -aiohttp-fast-url-dispatcher==0.3.0 aiohttp-fast-zlib==0.1.1 aiohttp==3.10.0b1 aiohttp_cors==0.7.0 diff --git a/pyproject.toml b/pyproject.toml index 6231539f7bc1f2..54a790703ab077 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ dependencies = [ "aiodns==3.2.0", "aiohttp==3.10.0b1", "aiohttp_cors==0.7.0", - "aiohttp-fast-url-dispatcher==0.3.0", "aiohttp-fast-zlib==0.1.1", "aiozoneinfo==0.2.1", "astral==2.2", diff --git a/requirements.txt b/requirements.txt index 674b31af7b422a..6f6a11b03a1467 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ aiodns==3.2.0 aiohttp==3.10.0b1 aiohttp_cors==0.7.0 -aiohttp-fast-url-dispatcher==0.3.0 aiohttp-fast-zlib==0.1.1 aiozoneinfo==0.2.1 astral==2.2 diff --git a/script/licenses.py b/script/licenses.py index 52e8fd0f3e7f88..358e0e03791544 100644 --- a/script/licenses.py +++ b/script/licenses.py @@ -125,7 +125,6 @@ def from_dict(cls, data: dict[str, str]) -> PackageDefinition: "aiocomelit", # https://github.com/chemelli74/aiocomelit/pull/138 "aioecowitt", # https://github.com/home-assistant-libs/aioecowitt/pull/180 "aiohappyeyeballs", # PSF-2.0 license - "aiohttp-fast-url-dispatcher", # https://github.com/bdraco/aiohttp-fast-url-dispatcher/pull/10 "aioopenexchangerates", # https://github.com/MartinHjelmare/aioopenexchangerates/pull/94 "aiooui", # https://github.com/Bluetooth-Devices/aiooui/pull/8 "aioruuvigateway", # https://github.com/akx/aioruuvigateway/pull/6 From 2b98b9e7b71148447c57e057605ee2391d52614d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 22 Jul 2024 14:04:31 -0500 Subject: [PATCH 03/10] fix exception --- tests/components/websocket_api/test_http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/components/websocket_api/test_http.py b/tests/components/websocket_api/test_http.py index 794dd410661db4..11665da11b46e3 100644 --- a/tests/components/websocket_api/test_http.py +++ b/tests/components/websocket_api/test_http.py @@ -5,7 +5,7 @@ from typing import Any, cast from unittest.mock import patch -from aiohttp import ServerDisconnectedError, WSMsgType, web +from aiohttp import WSMsgType, WSServerHandshakeError, web import pytest from homeassistant.components.websocket_api import ( @@ -374,7 +374,7 @@ async def test_prepare_fail( "homeassistant.components.websocket_api.http.web.WebSocketResponse.prepare", side_effect=(TimeoutError, web.WebSocketResponse.prepare), ), - pytest.raises(ServerDisconnectedError), + pytest.raises(WSServerHandshakeError), ): await hass_ws_client(hass) From 8fe1b529b97c1756c77338d3c2d13df916fffa6f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 22 Jul 2024 17:11:54 -0500 Subject: [PATCH 04/10] remove unused --- homeassistant/helpers/aiohttp_client.py | 5 +- homeassistant/helpers/backports/__init__.py | 1 - .../helpers/backports/aiohttp_resolver.py | 116 ------------------ 3 files changed, 3 insertions(+), 119 deletions(-) delete mode 100644 homeassistant/helpers/backports/__init__.py delete mode 100644 homeassistant/helpers/backports/aiohttp_resolver.py diff --git a/homeassistant/helpers/aiohttp_client.py b/homeassistant/helpers/aiohttp_client.py index 5c4ead4e6115e2..53b15907d0cbda 100644 --- a/homeassistant/helpers/aiohttp_client.py +++ b/homeassistant/helpers/aiohttp_client.py @@ -5,6 +5,7 @@ import asyncio from collections.abc import Awaitable, Callable from contextlib import suppress +import socket from ssl import SSLContext import sys from types import MappingProxyType @@ -13,6 +14,7 @@ import aiohttp from aiohttp import web from aiohttp.hdrs import CONTENT_TYPE, USER_AGENT +from aiohttp.resolver import AsyncResolver from aiohttp.web_exceptions import HTTPBadGateway, HTTPGatewayTimeout from homeassistant import config_entries @@ -23,7 +25,6 @@ from homeassistant.util.hass_dict import HassKey from homeassistant.util.json import json_loads -from .backports.aiohttp_resolver import AsyncResolver from .frame import warn_use from .json import json_dumps @@ -300,7 +301,7 @@ def _async_get_connector( ssl_context = ssl_util.get_default_no_verify_context() connector = aiohttp.TCPConnector( - family=family, + family=socket.AddressFamily(family), enable_cleanup_closed=ENABLE_CLEANUP_CLOSED, ssl=ssl_context, limit=MAXIMUM_CONNECTIONS, diff --git a/homeassistant/helpers/backports/__init__.py b/homeassistant/helpers/backports/__init__.py deleted file mode 100644 index e672fe1d3d2e97..00000000000000 --- a/homeassistant/helpers/backports/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Backports for helpers.""" diff --git a/homeassistant/helpers/backports/aiohttp_resolver.py b/homeassistant/helpers/backports/aiohttp_resolver.py deleted file mode 100644 index efa4ba4bb85934..00000000000000 --- a/homeassistant/helpers/backports/aiohttp_resolver.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Backport of aiohttp's AsyncResolver for Home Assistant. - -This is a backport of the AsyncResolver class from aiohttp 3.10. - -Before aiohttp 3.10, on system with IPv6 support, AsyncResolver would not fallback -to providing A records when AAAA records were not available. - -Additionally, unlike the ThreadedResolver, AsyncResolver -did not handle link-local addresses correctly. -""" - -from __future__ import annotations - -import asyncio -import socket -import sys -from typing import Any, TypedDict - -import aiodns -from aiohttp.abc import AbstractResolver - -# This is a backport of https://github.com/aio-libs/aiohttp/pull/8270 -# This can be removed once aiohttp 3.10 is the minimum supported version. - -_NUMERIC_SOCKET_FLAGS = socket.AI_NUMERICHOST | socket.AI_NUMERICSERV -_SUPPORTS_SCOPE_ID = sys.version_info >= (3, 9, 0) - - -class ResolveResult(TypedDict): - """Resolve result. - - This is the result returned from an AbstractResolver's - resolve method. - - :param hostname: The hostname that was provided. - :param host: The IP address that was resolved. - :param port: The port that was resolved. - :param family: The address family that was resolved. - :param proto: The protocol that was resolved. - :param flags: The flags that were resolved. - """ - - hostname: str - host: str - port: int - family: int - proto: int - flags: int - - -class AsyncResolver(AbstractResolver): - """Use the `aiodns` package to make asynchronous DNS lookups.""" - - def __init__(self, *args: Any, **kwargs: Any) -> None: - """Initialize the resolver.""" - if aiodns is None: - raise RuntimeError("Resolver requires aiodns library") - - self._loop = asyncio.get_running_loop() - self._resolver = aiodns.DNSResolver(*args, loop=self._loop, **kwargs) # type: ignore[misc] - - async def resolve( # type: ignore[override] - self, host: str, port: int = 0, family: int = socket.AF_INET - ) -> list[ResolveResult]: - """Resolve a host name to an IP address.""" - try: - resp = await self._resolver.getaddrinfo( - host, - port=port, - type=socket.SOCK_STREAM, - family=family, # type: ignore[arg-type] - flags=socket.AI_ADDRCONFIG, - ) - except aiodns.error.DNSError as exc: - msg = exc.args[1] if len(exc.args) >= 1 else "DNS lookup failed" - raise OSError(msg) from exc - hosts: list[ResolveResult] = [] - for node in resp.nodes: - address: tuple[bytes, int] | tuple[bytes, int, int, int] = node.addr - family = node.family - if family == socket.AF_INET6: - if len(address) > 3 and address[3] and _SUPPORTS_SCOPE_ID: - # This is essential for link-local IPv6 addresses. - # LL IPv6 is a VERY rare case. Strictly speaking, we should use - # getnameinfo() unconditionally, but performance makes sense. - result = await self._resolver.getnameinfo( - (address[0].decode("ascii"), *address[1:]), - _NUMERIC_SOCKET_FLAGS, - ) - resolved_host = result.node - else: - resolved_host = address[0].decode("ascii") - port = address[1] - else: # IPv4 - assert family == socket.AF_INET - resolved_host = address[0].decode("ascii") - port = address[1] - hosts.append( - ResolveResult( - hostname=host, - host=resolved_host, - port=port, - family=family, - proto=0, - flags=_NUMERIC_SOCKET_FLAGS, - ) - ) - - if not hosts: - raise OSError("DNS lookup failed") - - return hosts - - async def close(self) -> None: - """Close the resolver.""" - self._resolver.cancel() From ae6000a084b5d444fea4ba9b565b2dbf06695020 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 22 Jul 2024 17:13:07 -0500 Subject: [PATCH 05/10] dep timeout fixes --- homeassistant/util/location.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/homeassistant/util/location.py b/homeassistant/util/location.py index 24c49c5427c4ec..c00cf88699e59a 100644 --- a/homeassistant/util/location.py +++ b/homeassistant/util/location.py @@ -163,7 +163,8 @@ async def _get_whoami(session: aiohttp.ClientSession) -> dict[str, Any] | None: """Query whoami.home-assistant.io for location data.""" try: resp = await session.get( - WHOAMI_URL_DEV if HA_VERSION.endswith("0.dev0") else WHOAMI_URL, timeout=30 + WHOAMI_URL_DEV if HA_VERSION.endswith("0.dev0") else WHOAMI_URL, + timeout=aiohttp.ClientTimeout(total=30), ) except (aiohttp.ClientError, TimeoutError): return None From 0c88b51f5f60c8abdf1f13ebb2bff6c817edad3e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 22 Jul 2024 17:13:44 -0500 Subject: [PATCH 06/10] dep timeout fixes --- homeassistant/components/auth/indieauth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/auth/indieauth.py b/homeassistant/components/auth/indieauth.py index 45de94d5a701a5..dec6767d807171 100644 --- a/homeassistant/components/auth/indieauth.py +++ b/homeassistant/components/auth/indieauth.py @@ -94,7 +94,7 @@ async def fetch_redirect_uris(hass: HomeAssistant, url: str) -> list[str]: try: async with ( aiohttp.ClientSession() as session, - session.get(url, timeout=5) as resp, + session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp, ): async for data in resp.content.iter_chunked(1024): parser.feed(data.decode()) From cc09e42f71da0f8d5b8ee092ce2e36624bbbcb2a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 22 Jul 2024 17:14:11 -0500 Subject: [PATCH 07/10] dep timeout fixes --- homeassistant/components/system_health/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/system_health/__init__.py b/homeassistant/components/system_health/__init__.py index ca1d4026ea9f09..ce80f6303d913d 100644 --- a/homeassistant/components/system_health/__init__.py +++ b/homeassistant/components/system_health/__init__.py @@ -235,7 +235,7 @@ async def async_check_can_reach_url( session = aiohttp_client.async_get_clientsession(hass) try: - await session.get(url, timeout=5) + await session.get(url, timeout=aiohttp.ClientTimeout(total=5)) except aiohttp.ClientError: data = {"type": "failed", "error": "unreachable"} except TimeoutError: From a929c3b7555abb64a09f654650f500e396805cfe Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 22 Jul 2024 17:15:58 -0500 Subject: [PATCH 08/10] types --- homeassistant/components/lifx_cloud/scene.py | 2 +- homeassistant/components/no_ip/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/lifx_cloud/scene.py b/homeassistant/components/lifx_cloud/scene.py index 8e7ab0c2d46f18..5cb821b97c66e6 100644 --- a/homeassistant/components/lifx_cloud/scene.py +++ b/homeassistant/components/lifx_cloud/scene.py @@ -49,7 +49,7 @@ async def async_setup_platform( try: httpsession = async_get_clientsession(hass) async with asyncio.timeout(timeout): - scenes_resp = await httpsession.get(url, headers=headers) + scenes_resp = await httpsession.get(url, headers=headers) # type: ignore[arg-type] except (TimeoutError, aiohttp.ClientError): _LOGGER.exception("Error on %s", url) diff --git a/homeassistant/components/no_ip/__init__.py b/homeassistant/components/no_ip/__init__.py index 9680464c9faa24..64a8fad9786e99 100644 --- a/homeassistant/components/no_ip/__init__.py +++ b/homeassistant/components/no_ip/__init__.py @@ -101,7 +101,7 @@ async def _update_no_ip( try: async with asyncio.timeout(timeout): - resp = await session.get(url, params=params, headers=headers) + resp = await session.get(url, params=params, headers=headers) # type: ignore[arg-type] body = await resp.text() if body.startswith(("good", "nochg")): From 9f80b7b0403c884c390c0d206a7845101644df7c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 22 Jul 2024 17:18:13 -0500 Subject: [PATCH 09/10] dep timeout fixes --- homeassistant/components/amcrest/camera.py | 5 ++++- homeassistant/components/buienradar/camera.py | 4 +++- homeassistant/components/discord/notify.py | 3 ++- homeassistant/components/trafikverket_camera/coordinator.py | 5 ++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/amcrest/camera.py b/homeassistant/components/amcrest/camera.py index a55f9c81e64091..b9b2701eac6543 100644 --- a/homeassistant/components/amcrest/camera.py +++ b/homeassistant/components/amcrest/camera.py @@ -8,6 +8,7 @@ import logging from typing import TYPE_CHECKING, Any +import aiohttp from aiohttp import web from amcrest import AmcrestError from haffmpeg.camera import CameraMjpeg @@ -244,7 +245,9 @@ async def handle_async_mjpeg_stream( websession = async_get_clientsession(self.hass) streaming_url = self._api.mjpeg_url(typeno=self._resolution) stream_coro = websession.get( - streaming_url, auth=self._token, timeout=CAMERA_WEB_SESSION_TIMEOUT + streaming_url, + auth=self._token, + timeout=aiohttp.ClientTimeout(total=CAMERA_WEB_SESSION_TIMEOUT), ) return await async_aiohttp_proxy_web(self.hass, request, stream_coro) diff --git a/homeassistant/components/buienradar/camera.py b/homeassistant/components/buienradar/camera.py index 72bf6b7a3ebb57..e9a7d2517cb66d 100644 --- a/homeassistant/components/buienradar/camera.py +++ b/homeassistant/components/buienradar/camera.py @@ -115,7 +115,9 @@ async def __retrieve_radar_image(self) -> bool: headers = {} try: - async with session.get(url, timeout=5, headers=headers) as res: + async with session.get( + url, timeout=aiohttp.ClientTimeout(total=5), headers=headers + ) as res: res.raise_for_status() if res.status == 304: diff --git a/homeassistant/components/discord/notify.py b/homeassistant/components/discord/notify.py index c574b4583339e9..8a98d17291364d 100644 --- a/homeassistant/components/discord/notify.py +++ b/homeassistant/components/discord/notify.py @@ -7,6 +7,7 @@ import os.path from typing import Any, cast +import aiohttp import nextcord from nextcord.abc import Messageable @@ -81,7 +82,7 @@ async def async_get_file_from_url( async with session.get( url, ssl=verify_ssl, - timeout=30, + timeout=aiohttp.ClientTimeout(total=30), raise_for_status=True, ) as resp: content_length = resp.headers.get("Content-Length") diff --git a/homeassistant/components/trafikverket_camera/coordinator.py b/homeassistant/components/trafikverket_camera/coordinator.py index 8ead479fd1ce59..7bc5c556c00b9a 100644 --- a/homeassistant/components/trafikverket_camera/coordinator.py +++ b/homeassistant/components/trafikverket_camera/coordinator.py @@ -8,6 +8,7 @@ import logging from typing import TYPE_CHECKING +import aiohttp from pytrafikverket.exceptions import ( InvalidAuthentication, MultipleCamerasFound, @@ -77,7 +78,9 @@ async def _async_update_data(self) -> CameraData: if camera_data.fullsizephoto: image_url = f"{camera_data.photourl}?type=fullsize" - async with self.session.get(image_url, timeout=10) as get_image: + async with self.session.get( + image_url, timeout=aiohttp.ClientTimeout(total=10) + ) as get_image: if get_image.status not in range(200, 299): raise UpdateFailed("Could not retrieve image") image = BytesIO(await get_image.read()).getvalue() From 6ef71918a56499af021553ed5077b4549c143792 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 23 Jul 2024 08:39:00 -0500 Subject: [PATCH 10/10] Apply suggestions from code review --- homeassistant/components/lifx_cloud/scene.py | 2 +- homeassistant/components/no_ip/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/lifx_cloud/scene.py b/homeassistant/components/lifx_cloud/scene.py index 5d2441710007e8..b40cb081ed7d6c 100644 --- a/homeassistant/components/lifx_cloud/scene.py +++ b/homeassistant/components/lifx_cloud/scene.py @@ -49,7 +49,7 @@ async def async_setup_platform( try: httpsession = async_get_clientsession(hass) async with asyncio.timeout(timeout): - scenes_resp = await httpsession.get(url, headers=headers) # type: ignore[arg-type] + scenes_resp = await httpsession.get(url, headers=headers) except (TimeoutError, aiohttp.ClientError): _LOGGER.exception("Error on %s", url) diff --git a/homeassistant/components/no_ip/__init__.py b/homeassistant/components/no_ip/__init__.py index 6df727596e38bf..cb02490ac08d58 100644 --- a/homeassistant/components/no_ip/__init__.py +++ b/homeassistant/components/no_ip/__init__.py @@ -101,7 +101,7 @@ async def _update_no_ip( try: async with asyncio.timeout(timeout): - resp = await session.get(url, params=params, headers=headers) # type: ignore[arg-type] + resp = await session.get(url, params=params, headers=headers) body = await resp.text() if body.startswith(("good", "nochg")):