From 329a8a6d4d2f2b76495721cde57fac000c9f4957 Mon Sep 17 00:00:00 2001 From: Latent Logic Date: Sun, 15 Oct 2023 12:57:32 -0700 Subject: [PATCH 1/3] Handle broken aiohttp websocket in chat bot client It turns out that aiohttp WebSocket code does not recognize if the underlying connection is broken: https://github.com/aio-libs/aiohttp/issues/2309 As a twitch client we know we'll at minimum receive a PING request every 5 minutes even if no other channel updates are happening: https://dev.twitch.tv/docs/irc/#keepalive-messages (I tested this and saw every a ping request always happened between every 4min 1sec to 4min 51sec) If we miss a PING and fail to respond twitch considers the connection closed. Thus we can add a timeout to our websocket receive() call and if we've heard nothing in over 5 minutes our connection is broken and we should perform a reconnect. This has been tested running the bot, and then trying 3 different types of connection breaking: 1) Disabling the network interface on my computer for 5 minutes 2) Suspending laptop and resuming later 3) Dropping related active NAT state on router (similar to rebooting router) --- twitchAPI/chat/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/twitchAPI/chat/__init__.py b/twitchAPI/chat/__init__.py index 1f7eedf..99dcf2c 100644 --- a/twitchAPI/chat/__init__.py +++ b/twitchAPI/chat/__init__.py @@ -894,7 +894,18 @@ async def __task_receive(self): 'USERSTATE': self._handle_user_state } while not self.__connection.closed: - message = await self.__connection.receive() + try: # At minimum we should receive a PING request just under every 5 minutes + message = await self.__connection.receive(timeout=5*60) + except asyncio.TimeoutError: + self.logger.warning(f"Reached timeout for websocket receive, will attempt a reconnect") + if self.__running: + try: + await self._handle_base_reconnect() + except TwitchBackendException: + self.logger.exception('Connection to chat websocket lost and unable to reestablish connection!') + break + else: + break if message.type == aiohttp.WSMsgType.TEXT: messages = message.data.split('\r\n') for m in messages: From eb54a7bc47328bc0cbbc17e328b4b88b1d314d93 Mon Sep 17 00:00:00 2001 From: Latent Logic Date: Mon, 16 Oct 2023 19:00:11 -0700 Subject: [PATCH 2/3] Allow configuration of the no-message reset timeout --- twitchAPI/chat/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/twitchAPI/chat/__init__.py b/twitchAPI/chat/__init__.py index 99dcf2c..7acb621 100644 --- a/twitchAPI/chat/__init__.py +++ b/twitchAPI/chat/__init__.py @@ -544,7 +544,8 @@ def __init__(self, connection_url: Optional[str] = None, is_verified_bot: bool = False, initial_channel: Optional[List[str]] = None, - callback_loop: Optional[asyncio.AbstractEventLoop] = None): + callback_loop: Optional[asyncio.AbstractEventLoop] = None, + no_message_reset_time: int = 10): """ :param twitch: A Authenticated twitch instance :param connection_url: alternative connection url |default|:code:`None` @@ -553,6 +554,9 @@ def __init__(self, :param callback_loop: The asyncio eventloop to be used for callbacks. \n Set this if you or a library you use cares about which asyncio event loop is running the callbacks. Defaults to the one used by Chat. + :param no_message_reset_time: How many minutes of mo messages from Twitch before the connection is considered + dead. Twitch sends a PING just under every 5 minutes and the bot must respond to them for Twitch to keep + the connection active. At 10 minutes we've definitely missed at least one PING |default|:code:`10` """ self.logger: Logger = getLogger('twitchAPI.chat') """The logger used for Chat related log messages""" @@ -568,6 +572,7 @@ def __init__(self, self.ping_jitter: int = 4 """Jitter in seconds for ping messages. This should usually not be changed.""" self._callback_loop = callback_loop + self.no_message_reset_time: int = no_message_reset_time self.listen_confirm_timeout: int = 30 """Time in second that any :code:`listen_` should wait for its subscription to be completed.""" self.reconnect_delay_steps: List[int] = [0, 1, 2, 4, 8, 16, 32, 64, 128] @@ -895,7 +900,7 @@ async def __task_receive(self): } while not self.__connection.closed: try: # At minimum we should receive a PING request just under every 5 minutes - message = await self.__connection.receive(timeout=5*60) + message = await self.__connection.receive(timeout=self.no_message_reset_time*60) except asyncio.TimeoutError: self.logger.warning(f"Reached timeout for websocket receive, will attempt a reconnect") if self.__running: From c46c0be6b730cf4343491c9e49183d07efd99802 Mon Sep 17 00:00:00 2001 From: Latent Logic Date: Tue, 17 Oct 2023 11:01:20 -0700 Subject: [PATCH 3/3] Allow receive timeout to be None for infinite hang --- twitchAPI/chat/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/twitchAPI/chat/__init__.py b/twitchAPI/chat/__init__.py index 7acb621..0ebf251 100644 --- a/twitchAPI/chat/__init__.py +++ b/twitchAPI/chat/__init__.py @@ -545,7 +545,7 @@ def __init__(self, is_verified_bot: bool = False, initial_channel: Optional[List[str]] = None, callback_loop: Optional[asyncio.AbstractEventLoop] = None, - no_message_reset_time: int = 10): + no_message_reset_time: Optional[float] = 10): """ :param twitch: A Authenticated twitch instance :param connection_url: alternative connection url |default|:code:`None` @@ -572,7 +572,7 @@ def __init__(self, self.ping_jitter: int = 4 """Jitter in seconds for ping messages. This should usually not be changed.""" self._callback_loop = callback_loop - self.no_message_reset_time: int = no_message_reset_time + self.no_message_reset_time: Optional[float] = no_message_reset_time self.listen_confirm_timeout: int = 30 """Time in second that any :code:`listen_` should wait for its subscription to be completed.""" self.reconnect_delay_steps: List[int] = [0, 1, 2, 4, 8, 16, 32, 64, 128] @@ -881,6 +881,7 @@ async def _send_message(self, message: str): await self.__connection.send_str(message) async def __task_receive(self): + receive_timeout = None if self.no_message_reset_time is None else self.no_message_reset_time * 60 try: handlers: Dict[str, Callable] = { 'PING': self._handle_ping, @@ -900,7 +901,7 @@ async def __task_receive(self): } while not self.__connection.closed: try: # At minimum we should receive a PING request just under every 5 minutes - message = await self.__connection.receive(timeout=self.no_message_reset_time*60) + message = await self.__connection.receive(timeout=receive_timeout) except asyncio.TimeoutError: self.logger.warning(f"Reached timeout for websocket receive, will attempt a reconnect") if self.__running: