Cache CryptoUtil.getkey (redshiftzero's idea)

Adds caching to CryptoUtil.getkey, to reduce the number of expensive
GPG key lookup operations. It uses CryptoUtil.keycache, an
OrderedDict, so we can push out old items once we reach the cache size
limit. Using functools.lru_cache would have taken care of that, but
meant we couldn't avoid caching sources without keys, so delays in key
generation would mean the source key would be unusable until the
server were restarted.

The cache is primed in securedrop/journalist.py to avoid cold starts.

securedrop/crypto_util.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import collections
 from distutils.version import StrictVersion
 import pretty_bad_protocol as gnupg
 import os
@@ -72,6 +73,9 @@ class CryptoUtil:
     # to set an expiration date.
     DEFAULT_KEY_EXPIRATION_DATE = '0'
 
+    keycache = collections.OrderedDict()  # type: collections.OrderedDict
+    keycache_limit = 1000
+
     def __init__(self,
                  scrypt_params,
                  scrypt_id_pepper,
@@ -224,12 +228,21 @@ def delete_reply_keypair(self, source_filesystem_id):
         temp_gpg = gnupg.GPG(binary='gpg2', homedir=self.gpg_key_dir)
         # The subkeys keyword argument deletes both secret and public keys.
         temp_gpg.delete_keys(key, secret=True, subkeys=True)
+        del self.keycache[source_filesystem_id]
 
     def getkey(self, name):
+        if name in self.keycache:
+            return self.keycache[name]
+
         for key in self.gpg.list_keys():
             for uid in key['uids']:
                 if name in uid:
-                    return key['fingerprint']
+                    fingerprint = key['fingerprint']
+                    self.keycache[name] = fingerprint
+                    if len(self.keycache) > self.keycache_limit:
+                        self.keycache.popitem(last=False)
+                    return fingerprint
+
         return None
 
     def export_pubkey(self, name):

securedrop/journalist.py

@@ -3,10 +3,25 @@
 from sdconfig import config
 
 from journalist_app import create_app
+from models import Source
+from source_app.utils import asynchronous
 
 app = create_app(config)
 
 
+@asynchronous
+def prime_keycache():
+    """
+    Preloads CryptoUtil.keycache.
+    """
+    with app.app_context():
+        for source in Source.query.filter_by(pending=False).all():
+            app.crypto_util.getkey(source.filesystem_id)
+
+
+prime_keycache()
+
+
 if __name__ == "__main__":  # pragma: no cover
     debug = getattr(config, 'env', 'prod') != 'prod'
     app.run(debug=debug, host='0.0.0.0', port=8081)  # nosec