From 4c360e2718d6e5c9de64f49ddf588a862057f91e Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Wed, 15 May 2024 06:56:54 +1000 Subject: [PATCH 1/3] Rate limit requests for addons.xml.gz Rate limiting has been added to mirrors.kodi.tv which the multiple requests for the various repository addons.xml.gz will trigger. This adds some rate limiting, error handling and retry backoff to avoid the rate limit resulting in the addon checker failing --- kodi_addon_checker/addons/Repository.py | 51 +++++++++++++++++++++---- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/kodi_addon_checker/addons/Repository.py b/kodi_addon_checker/addons/Repository.py index 5ba1813..a71068c 100644 --- a/kodi_addon_checker/addons/Repository.py +++ b/kodi_addon_checker/addons/Repository.py @@ -6,7 +6,9 @@ See LICENSES/README.md for more information. """ +import atexit import gzip +import time import xml.etree.ElementTree as ET from io import BytesIO @@ -16,26 +18,59 @@ from ..versions import AddonVersion +class RateLimitedAdapter(requests.adapters.HTTPAdapter): + def __init__(self, *args, retries=5, wait=10, **kwargs): + self._last_send = None + self._wait_time = wait + max_retries = requests.adapters.Retry( + total=retries, + backoff_factor=wait, + status_forcelist={429, }, + allowed_methods=None, + ) + kwargs.setdefault('max_retries', max_retries) + super().__init__(*args, **kwargs) + + def send(self, *args, **kwargs): + if self._last_send: + delta = time.time() - self._last_send + if delta < self._wait_time: + time.sleep(self._wait_time - delta) + + self._last_send = time.time() + response = super().send(*args, **kwargs) + status_code = getattr(response, 'status_code', None) + if 300 <= status_code < 400: + self._last_send = None + return response + + class Repository(): + # Recover from unreliable mirrors + _session = requests.Session() + _adapter = RateLimitedAdapter(retries=5, wait=10) + _session.mount('http://', _adapter) + _session.mount('https://', _adapter) + atexit.register(_session.close) + def __init__(self, version, path): super().__init__() self.version = version self.path = path + self.addons = [] - # Recover from unreliable mirrors - session = requests.Session() - adapter = requests.adapters.HTTPAdapter(max_retries=5) - session.mount('http://', adapter) - session.mount('https://', adapter) - - content = session.get(path, timeout=(30, 30)).content + try: + response = self._session.get(path, timeout=(30, 30)) + response.raise_for_status() + except requests.exceptions.RequestException as exc: + return + content = response.content if path.endswith('.gz'): with gzip.open(BytesIO(content), 'rb') as xml_file: content = xml_file.read() tree = ET.fromstring(content) - self.addons = [] for addon in tree.findall("addon"): self.addons.append(Addon(addon)) From a9b0b1f1ace2e536e607ade330b29c32180978b6 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Fri, 24 May 2024 11:18:15 +1000 Subject: [PATCH 2/3] Limit pool connections and block rather than explicit time delays --- kodi_addon_checker/addons/Repository.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kodi_addon_checker/addons/Repository.py b/kodi_addon_checker/addons/Repository.py index a71068c..c6f6faf 100644 --- a/kodi_addon_checker/addons/Repository.py +++ b/kodi_addon_checker/addons/Repository.py @@ -24,7 +24,7 @@ def __init__(self, *args, retries=5, wait=10, **kwargs): self._wait_time = wait max_retries = requests.adapters.Retry( total=retries, - backoff_factor=wait, + backoff_factor=wait or 10, status_forcelist={429, }, allowed_methods=None, ) @@ -32,7 +32,7 @@ def __init__(self, *args, retries=5, wait=10, **kwargs): super().__init__(*args, **kwargs) def send(self, *args, **kwargs): - if self._last_send: + if self._wait_time and self._last_send: delta = time.time() - self._last_send if delta < self._wait_time: time.sleep(self._wait_time - delta) @@ -48,7 +48,7 @@ def send(self, *args, **kwargs): class Repository(): # Recover from unreliable mirrors _session = requests.Session() - _adapter = RateLimitedAdapter(retries=5, wait=10) + _adapter = RateLimitedAdapter(retries=5, wait=None, pool_maxsize=3, pool_block=True) _session.mount('http://', _adapter) _session.mount('https://', _adapter) atexit.register(_session.close) @@ -57,7 +57,6 @@ def __init__(self, version, path): super().__init__() self.version = version self.path = path - self.addons = [] try: response = self._session.get(path, timeout=(30, 30)) @@ -70,6 +69,7 @@ def __init__(self, version, path): with gzip.open(BytesIO(content), 'rb') as xml_file: content = xml_file.read() + self.addons = [] tree = ET.fromstring(content) for addon in tree.findall("addon"): self.addons.append(Addon(addon)) From 7b9036fec0856340605c50e097fc416bdbefb604 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Fri, 24 May 2024 11:38:15 +1000 Subject: [PATCH 3/3] Update default value to None for wait parameter of RateLimitedAdapter --- kodi_addon_checker/addons/Repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kodi_addon_checker/addons/Repository.py b/kodi_addon_checker/addons/Repository.py index c6f6faf..ec19509 100644 --- a/kodi_addon_checker/addons/Repository.py +++ b/kodi_addon_checker/addons/Repository.py @@ -19,7 +19,7 @@ class RateLimitedAdapter(requests.adapters.HTTPAdapter): - def __init__(self, *args, retries=5, wait=10, **kwargs): + def __init__(self, *args, retries=5, wait=None, **kwargs): self._last_send = None self._wait_time = wait max_retries = requests.adapters.Retry( @@ -48,7 +48,7 @@ def send(self, *args, **kwargs): class Repository(): # Recover from unreliable mirrors _session = requests.Session() - _adapter = RateLimitedAdapter(retries=5, wait=None, pool_maxsize=3, pool_block=True) + _adapter = RateLimitedAdapter(retries=5, pool_maxsize=3, pool_block=True) _session.mount('http://', _adapter) _session.mount('https://', _adapter) atexit.register(_session.close)