Add exponential backoff extension

.nengobones.yml

@@ -38,6 +38,7 @@ setup_py:
   include_package_data: True
   install_req:
     - sphinx>=1.8
+    - backoff>=1.10.0
   docs_req:
     - jupyter
     - matplotlib

CHANGES.rst

@@ -28,6 +28,9 @@ Release History
   ``nengo_sphinx_theme.ext.autoautosummary``, which allows classes/functions
   documented with ``autoautosummary`` or ``automodule`` to be moved to a different
   nominal namespace. (`#52 <https://github.com/nengo/nengo-sphinx-theme/pull/52>`__)
+- Added ``nengo_sphinx_theme.ext.backoff``, which monkeypatches the Sphinx
+  HTTP request functionality to add exponential backoff.
+  (`#53 <https://github.com/nengo/nengo-sphinx-theme/pull/53>`__)
 
 1.2.0 (November 14, 2019)
 =========================

docs/conf.py

@@ -17,6 +17,7 @@
     "sphinx.ext.viewcode",
     "nbsphinx",
     "nengo_sphinx_theme",
+    "nengo_sphinx_theme.ext.backoff",
     "nengo_sphinx_theme.ext.redirects",
     "numpydoc",
     "nengo_sphinx_theme.ext.resolvedefaults",
@@ -72,6 +73,7 @@
 linkcheck_anchors = True
 default_role = "py:obj"
 pygments_style = "sphinx"
+user_agent = "nengo_sphinx_theme"
 
 project = "Nengo Sphinx Theme"
 authors = "Applied Brain Research"

docs/extensions.rst

@@ -48,3 +48,8 @@ Redirects
 
 The following page should redirect to the
 `deeply nested testing page <redirect/to/nested-page.html>`_.
+
+Backoff
+=======
+
+.. automodule:: nengo_sphinx_theme.ext.backoff

nengo_sphinx_theme/ext/backoff.py

@@ -0,0 +1,33 @@
+"""
+Monkeypatches the Sphinx HTTP request functions to add exponential backoff.
+
+The main use case for this is to avoid server-side rejections caused by too many
+requests when running the linkchecker.
+"""
+
+import requests
+
+import backoff
+from sphinx.util import requests as sphinx_requests
+from sphinx.util.requests import get, head
+
+
+@backoff.on_predicate(backoff.expo, lambda x: x.status_code == 429, max_time=60)
+@backoff.on_exception(
+    backoff.constant, requests.exceptions.RequestException, max_tries=3,
+)
+def get_with_backoff(*args, **kwargs):
+    return get(*args, **kwargs)
+
+
+@backoff.on_predicate(backoff.expo, lambda x: x.status_code == 429, max_time=60)
+@backoff.on_exception(
+    backoff.constant, requests.exceptions.RequestException, max_tries=3,
+)
+def head_with_backoff(*args, **kwargs):
+    return head(*args, **kwargs)
+
+
+def setup(_):
+    sphinx_requests.get = get_with_backoff
+    sphinx_requests.head = head_with_backoff

setup.py

@@ -33,6 +33,7 @@ def read(*filenames, **kwargs):
 
 install_req = [
     "sphinx>=1.8",
+    "backoff>=1.10.0",
 ]
 docs_req = [
     "jupyter",