Merge pull request #269 from jmcarp/feature/respect-caching-headers

Respect caching headers from API.

__init__.py

@@ -15,14 +15,13 @@
 from openfecwebapp import utils
 from openfecwebapp import config
 from openfecwebapp.views import render_search_results, render_candidate, render_committee
-from openfecwebapp.api_caller import load_search_results, load_with_nested, install_cache
+from openfecwebapp.api_caller import load_search_results, load_with_nested
 
 import jinja2
 import json
 import locale
 import logging
 import re
-import sys
 
 
 locale.setlocale(locale.LC_ALL, '')
@@ -273,7 +272,5 @@ def restrict_cycles(value):
 
 
 if __name__ == '__main__':
-    if '--cached' in sys.argv:
-        install_cache()
     files = ['./rev-manifest.json']
     app.run(host=config.host, port=int(config.port), debug=config.debug, extra_files=files)

manifest_prod.yml

@@ -16,3 +16,4 @@ env:
   FEC_WEB_API_URL: https://api.open.fec.gov/
   FEC_WEB_SERVER_NAME: open.fec.gov
   FEC_FORCE_HTTPS: true
+  FEC_WEB_CACHE: true

openfecwebapp/api_caller.py

@@ -2,27 +2,31 @@
 from urllib import parse
 
 import requests
+import cachecontrol
 
 from openfecwebapp import utils
-from openfecwebapp.config import api_location, api_version, api_key
+from openfecwebapp import config
 
 
 MAX_FINANCIALS_COUNT = 4
 
 
+session = requests.Session()
+
+if config.cache:
+    cachecontrol.CacheControl(session, cache=utils.LRUCache(config.cache_size))
+
+
 def _call_api(*path_parts, **filters):
-    if api_key:
-        filters['api_key'] = api_key
+    if config.api_key:
+        filters['api_key'] = config.api_key
 
-    path = os.path.join(api_version, *[x.strip('/') for x in path_parts])
-    url = parse.urljoin(api_location, path)
+    path = os.path.join(config.api_version, *[x.strip('/') for x in path_parts])
+    url = parse.urljoin(config.api_location, path)
 
-    results = requests.get(url, params=filters)
+    results = session.get(url, params=filters)
 
-    if results.status_code == requests.codes.ok:
-        return results.json()
-    else:
-        return {}
+    return results.json() if results.ok else {}
 
 
 def load_search_results(query, query_type='candidates'):
@@ -69,8 +73,3 @@ def load_cmte_financials(committee_id, **filters):
         'reports': reports['results'],
         'totals': totals['results'],
     }
-
-
-def install_cache():
-    import requests_cache
-    requests_cache.install_cache()

openfecwebapp/config.py

@@ -9,6 +9,8 @@
 api_key = os.getenv('FEC_WEB_API_KEY', '')
 api_key_public = os.getenv('FEC_WEB_API_KEY_PUBLIC', '')
 server_name = os.getenv('FEC_WEB_SERVER_NAME')
+cache = os.getenv('FEC_WEB_CACHE')
+cache_size = int(os.getenv('FEC_WEB_CACHE_SIZE', 1000))
 
 # the username and password should be the same for both the
 # web app and API

openfecwebapp/utils.py

@@ -1,6 +1,30 @@
 import datetime
+import threading
+
+import cachetools
+import cachecontrol
 
 
 def current_cycle():
     year = datetime.datetime.now().year
     return year + year % 2
+
+
+class LRUCache(cachecontrol.cache.BaseCache):
+    """A thread-safe least recently updated cache adapted to work with
+    Cache-Control.
+    """
+    def __init__(self, maxsize):
+        self.lock = threading.Lock()
+        self.data = cachetools.LRUCache(maxsize)
+
+    def get(self, key):
+        return self.data.get(key, None)
+
+    def set(self, key, value):
+        with self.lock:
+            self.data[key] = value
+
+    def delete(self, key):
+        with self.lock:
+            self.data.clear()

requirements.txt

@@ -9,11 +9,12 @@ pytest>=2.7.0
 pytest-cov>=1.8.1
 python-dateutil>=2.4.2
 requests==2.5.1
-requests-cache==0.4.9
 selenium>=2.45.0
 newrelic==2.50.0.39
 gunicorn
 webargs>=0.13.0
 Flask-SSLify
+CacheControl>=0.11.5
+cachetools>=1.0.1
 
 git+https://github.com/jmcarp/flask-compress@cache