Add optional caching of GitHub API requests

This seems to be the least invasive way of avoiding the overhead of
calling the GitHub API when developing the metrics. The most obvious
alternative would be persistently-memoizing the functions in the `query`
module (e.g. with `cachier`), but that would require more code.

.gitignore

@@ -6,3 +6,4 @@
 __pycache__/
 github.db
 htmlcov
+github-cache.sqlite

DEVELOPERS.md

@@ -71,7 +71,7 @@ e.g `just metrics prs` to run metrics/tasks/prs.py
 
 All tasks are defined in `metrics/tasks` and must have a `main()` function that takes no arguments.
 
-### Fast debug mode
+### Speeding up development
 
 You can set a flag to trigger a fast mode which only retrieves and handful of PRs
 but allows the main code paths to be tested quickly.
@@ -80,6 +80,23 @@ but allows the main code paths to be tested quickly.
 DEBUG_FAST=t just metrics prs
 ```
 
+Alternatively you can turn on caching of GitHub API requests.
+This is particularly useful when iterating on metric definitions
+without changing the data that we retrieve from the API.
+
+```
+DEBUG_CACHE=t just metrics prs
+```
+
+NB that the cache has no expiry time
+(although it will be bypassed on subsequent runs if `DEBUG_CACHE isn't defined).
+You can clear the cache explicitly.
+
+```
+just clean-cache
+```
+
+
 ## Tests
 Run the tests with:
 ```

justfile

@@ -168,3 +168,7 @@ docker-build env="dev": _env
 docker-run env="dev" *args="": _env
     {{ just_executable() }} docker-build {{ env }}
     docker compose run --rm metrics-{{ env }} {{ args }}
+
+# See DEVELOPERS.md
+clean-cache:
+    rm -f github-cache.sqlite

metrics/github/client.py

@@ -1,13 +1,22 @@
 import json
+import os
 import textwrap
 
 import requests
 import requests.utils
+import requests_cache
 import structlog
 
 
 log = structlog.get_logger()
 
+# See DEVELOPERS.md
+if "DEBUG_CACHE" in os.environ:
+    requests_cache.install_cache(
+        "github-cache",
+        # Turn on caching for POST requests because that's what GraphQL uses
+        allowable_methods=("GET", "HEAD", "POST"),
+    )
 
 session = requests.Session()
 

requirements.prod.in

@@ -1,5 +1,6 @@
   greenlet
   requests
+  requests-cache
   slack-bolt
   sqlalchemy[postgresql_psycopgbinary]
   structlog

requirements.prod.txt

@@ -4,6 +4,16 @@
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=requirements.prod.txt requirements.prod.in
 #
+attrs==24.2.0 \
+    --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \
+    --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2
+    # via
+    #   cattrs
+    #   requests-cache
+cattrs==24.1.2 \
+    --hash=sha256:67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0 \
+    --hash=sha256:8028cfe1ff5382df59dd36474a86e02d817b06eaf8af84555441bac915d2ef85
+    # via requests-cache
 certifi==2024.8.30 \
     --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \
     --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9
@@ -183,6 +193,10 @@ idna==3.10 \
     --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
     --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
     # via requests
+platformdirs==4.3.6 \
+    --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
+    --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
+    # via requests-cache
 psycopg[binary]==3.2.2 \
     --hash=sha256:8bad2e497ce22d556dac1464738cb948f8d6bab450d965cf1d8a8effd52412e0 \
     --hash=sha256:babf565d459d8f72fb65da5e211dd0b58a52c51e4e1fa9cadecff42d6b7619b2
@@ -256,11 +270,21 @@ psycopg-binary==3.2.2 \
 requests==2.32.3 \
     --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \
     --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
+    # via
+    #   -r requirements.prod.in
+    #   requests-cache
+requests-cache==1.2.1 \
+    --hash=sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603 \
+    --hash=sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1
     # via -r requirements.prod.in
 sentry-sdk==2.14.0 \
     --hash=sha256:1e0e2eaf6dad918c7d1e0edac868a7bf20017b177f242cefe2a6bcd47955961d \
     --hash=sha256:b8bc3dc51d06590df1291b7519b85c75e2ced4f28d9ea655b6d54033503b5bf4
     # via -r requirements.prod.in
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
+    # via url-normalize
 slack-bolt==1.20.1 \
     --hash=sha256:4657e592339797b9b804547a21e6b35dd8e2cd1eab676bfb23960660aae049fd \
     --hash=sha256:8fa26e72b0e55c18c1d34a73558e7fe2150bdc7c947de780b938fdb1d7e854fe
@@ -330,9 +354,14 @@ typing-extensions==4.12.2 \
     # via
     #   psycopg
     #   sqlalchemy
+url-normalize==1.4.3 \
+    --hash=sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2 \
+    --hash=sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed
+    # via requests-cache
 urllib3==2.2.3 \
     --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \
     --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9
     # via
     #   requests
+    #   requests-cache
     #   sentry-sdk