diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2 b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2 index 9043420258..4ca727a1e0 100644 --- a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2 +++ b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2 @@ -165,10 +165,10 @@ /var/www/securedrop/__pycache__/ rw, /var/www/securedrop/__pycache__/* rw, /var/www/securedrop/config.py r, - /var/www/securedrop/crypto_util.py r, /var/www/securedrop/db.py r, /var/www/securedrop/dictionaries/adjectives.txt r, /var/www/securedrop/dictionaries/nouns.txt r, + /var/www/securedrop/encryption.py r, /var/www/securedrop/execution.py r, /var/www/securedrop/i18n.py r, /var/www/securedrop/journalist.py r, diff --git a/securedrop/crypto_util.py b/securedrop/crypto_util.py deleted file mode 100644 index d6b80bfe2c..0000000000 --- a/securedrop/crypto_util.py +++ /dev/null @@ -1,221 +0,0 @@ -# -*- coding: utf-8 -*- - -from distutils.version import StrictVersion -from typing import Optional - -import pretty_bad_protocol as gnupg -import os -import re - -from datetime import date -from flask import current_app -from pretty_bad_protocol._util import _is_stream, _make_binary_stream -from redis import Redis -from typing import List - -from typing import Dict - -import rm - - -# monkey patch to work with Focal gnupg. -# https://github.com/isislovecruft/python-gnupg/issues/250 -from source_user import SourceUser - -gnupg._parsers.Verify.TRUST_LEVELS["DECRYPTION_COMPLIANCE_MODE"] = 23 - -# to fix GPG error #78 on production -os.environ['USERNAME'] = 'www-data' - - -def monkey_patch_delete_handle_status( - self: gnupg._parsers.DeleteResult, key: str, value: str -) -> None: - """ - Parse a status code from the attached GnuPG process. - :raises: :exc:`~exceptions.ValueError` if the status message is unknown. - """ - if key in ("DELETE_PROBLEM", "KEY_CONSIDERED"): - self.status = self.problem_reason.get(value, "Unknown error: %r" % value) - elif key in ("PINENTRY_LAUNCHED"): - self.status = key.replace("_", " ").lower() - else: - raise ValueError("Unknown status message: %r" % key) - - -# Monkey patching to resolve https://github.com/freedomofpress/securedrop/issues/4294 -gnupg._parsers.DeleteResult._handle_status = monkey_patch_delete_handle_status - - -class CryptoException(Exception): - pass - - -class CryptoUtil: - - GPG_KEY_TYPE = "RSA" - - # All reply keypairs will be "created" on the same day SecureDrop (then - # Strongbox) was publicly released for the first time. - # https://www.newyorker.com/news/news-desk/strongbox-and-aaron-swartz - DEFAULT_KEY_CREATION_DATE = date(2013, 5, 14) - - # '0' is the magic value that tells GPG's batch key generation not - # to set an expiration date. - DEFAULT_KEY_EXPIRATION_DATE = '0' - - REDIS_FINGERPRINT_HASH = "sd/crypto-util/fingerprints" - REDIS_KEY_HASH = "sd/crypto-util/keys" - - SOURCE_KEY_UID_RE = re.compile(r"(Source|Autogenerated) Key <[-A-Za-z0-9+/=_]+>") - - def __init__(self, - securedrop_root: str, - gpg_key_dir: str) -> None: - self.__securedrop_root = securedrop_root - - if os.environ.get('SECUREDROP_ENV') in ('dev', 'test'): - # Optimize crypto to speed up tests (at the expense of security - # DO NOT use these settings in production) - self.__gpg_key_length = 1024 - else: # pragma: no cover - self.__gpg_key_length = 4096 - - self.do_runtime_tests() - - # --pinentry-mode, required for SecureDrop on GPG 2.1.x+, was - # added in GPG 2.1. - self.gpg_key_dir = gpg_key_dir - gpg_binary = gnupg.GPG(binary='gpg2', homedir=self.gpg_key_dir) - if StrictVersion(gpg_binary.binary_version) >= StrictVersion('2.1'): - self.gpg = gnupg.GPG(binary='gpg2', - homedir=gpg_key_dir, - options=['--pinentry-mode loopback']) - else: - self.gpg = gpg_binary - - self.redis = Redis(decode_responses=True) - - # Make sure these pass before the app can run - def do_runtime_tests(self) -> None: - # crash if we don't have a way to securely remove files - if not rm.check_secure_delete_capability(): - raise AssertionError("Secure file deletion is not possible.") - - def genkeypair(self, source_user: SourceUser) -> gnupg._parsers.GenKey: - """Generate a GPG key through batch file key generation. - """ - genkey_obj = self.gpg.gen_key(self.gpg.gen_key_input( - key_type=self.GPG_KEY_TYPE, - key_length=self.__gpg_key_length, - passphrase=source_user.gpg_secret, - name_email=source_user.filesystem_id, - name_real="Source Key", - creation_date=self.DEFAULT_KEY_CREATION_DATE.isoformat(), - expire_date=self.DEFAULT_KEY_EXPIRATION_DATE - )) - return genkey_obj - - def find_source_key(self, fingerprint: str) -> Optional[Dict]: - """ - Searches the GPG keyring for a source key. - - A source key has the given fingerprint and is labeled either - "Source Key" or "Autogenerated Key". - - Returns the key or None. - """ - keys = self.gpg.list_keys() - for key in keys: - if fingerprint != key["fingerprint"]: - continue - - for uid in key["uids"]: - if self.SOURCE_KEY_UID_RE.match(uid): - return key - else: - return None - return None - - def delete_reply_keypair(self, source_filesystem_id: str) -> None: - fingerprint = self.get_fingerprint(source_filesystem_id) - - if not fingerprint: - return - - # verify that the key with the given fingerprint belongs to a source - key = self.find_source_key(fingerprint) - if not key: - raise ValueError("source key not found") - - # Always delete keys without invoking pinentry-mode = loopback - # see: https://lists.gnupg.org/pipermail/gnupg-users/2016-May/055965.html - temp_gpg = gnupg.GPG(binary='gpg2', homedir=self.gpg_key_dir, options=["--yes"]) - - # The subkeys keyword argument deletes both secret and public keys. - temp_gpg.delete_keys(fingerprint, secret=True, subkeys=True) - self.redis.hdel(self.REDIS_KEY_HASH, self.get_fingerprint(source_filesystem_id)) - self.redis.hdel(self.REDIS_FINGERPRINT_HASH, source_filesystem_id) - - def get_fingerprint(self, name: str) -> Optional[str]: - """ - Returns the fingerprint of the GPG key for the given name. - - The supplied name is usually a source filesystem ID. - """ - fingerprint = self.redis.hget(self.REDIS_FINGERPRINT_HASH, name) - if fingerprint: - return fingerprint - - for key in self.gpg.list_keys(): - for uid in key['uids']: - if name in uid: - self.redis.hset(self.REDIS_FINGERPRINT_HASH, name, key['fingerprint']) - return key['fingerprint'] - - return None - - def get_pubkey(self, name: str) -> Optional[str]: - """ - Returns the GPG public key for the given name. - - The supplied name is usually a source filesystem ID. - """ - fingerprint = self.get_fingerprint(name) - if not fingerprint: - return None - - key = self.redis.hget(self.REDIS_KEY_HASH, fingerprint) - if key: - return key - - key = self.gpg.export_keys(fingerprint) - self.redis.hset(self.REDIS_KEY_HASH, fingerprint, key) - return key - - def encrypt(self, plaintext: str, fingerprints: List[str], output: Optional[str] = None) -> str: - # Verify the output path - if output: - current_app.storage.verify(output) - - # Remove any spaces from provided fingerprints GPG outputs fingerprints - # with spaces for readability, but requires the spaces to be removed - # when using fingerprints to specify recipients. - fingerprints = [fpr.replace(' ', '') for fpr in fingerprints] - - if not _is_stream(plaintext): - plaintext = _make_binary_stream(plaintext, "utf_8") - - out = self.gpg.encrypt(plaintext, - *fingerprints, - output=output, - always_trust=True, - armor=False) - if out.ok: - return out.data - else: - raise CryptoException(out.stderr) - - def decrypt(self, source_user: SourceUser, ciphertext: bytes) -> str: - data = self.gpg.decrypt(ciphertext, passphrase=source_user.gpg_secret).data - return data.decode('utf-8') diff --git a/securedrop/encryption.py b/securedrop/encryption.py new file mode 100644 index 0000000000..0cd857b72e --- /dev/null +++ b/securedrop/encryption.py @@ -0,0 +1,258 @@ +import typing +from distutils.version import StrictVersion +from io import StringIO, BytesIO +from pathlib import Path +from typing import Optional, Dict, List + +import pretty_bad_protocol as gnupg +import os +import re + +from datetime import date +from redis import Redis + + +if typing.TYPE_CHECKING: + from source_user import SourceUser + + +def _monkey_patch_username_in_env() -> None: + # To fix https://github.com/freedomofpress/securedrop/issues/78 + os.environ["USERNAME"] = "www-data" + + +def _monkey_patch_unknown_status_message() -> None: + # To fix https://github.com/isislovecruft/python-gnupg/issues/250 with Focal gnupg + gnupg._parsers.Verify.TRUST_LEVELS["DECRYPTION_COMPLIANCE_MODE"] = 23 + + +def _monkey_patch_delete_handle_status() -> None: + # To fix https://github.com/freedomofpress/securedrop/issues/4294 + def _updated_handle_status(self: gnupg._parsers.DeleteResult, key: str, value: str) -> None: + """ + Parse a status code from the attached GnuPG process. + :raises: :exc:`~exceptions.ValueError` if the status message is unknown. + """ + if key in ("DELETE_PROBLEM", "KEY_CONSIDERED"): + self.status = self.problem_reason.get(value, "Unknown error: %r" % value) + elif key in ("PINENTRY_LAUNCHED"): + self.status = key.replace("_", " ").lower() + else: + raise ValueError("Unknown status message: %r" % key) + + gnupg._parsers.DeleteResult._handle_status = _updated_handle_status + + +def _setup_monkey_patches_for_gnupg() -> None: + _monkey_patch_username_in_env() + _monkey_patch_unknown_status_message() + _monkey_patch_delete_handle_status() + + +_setup_monkey_patches_for_gnupg() + + +class GpgKeyNotFoundError(Exception): + pass + + +class GpgEncryptError(Exception): + pass + + +class GpgDecryptError(Exception): + pass + + +_default_encryption_mgr: Optional["EncryptionManager"] = None + + +class EncryptionManager: + + GPG_KEY_TYPE = "RSA" + GPG_KEY_LENGTH = 4096 + + # All reply keypairs will be "created" on the same day SecureDrop (then + # Strongbox) was publicly released for the first time. + # https://www.newyorker.com/news/news-desk/strongbox-and-aaron-swartz + DEFAULT_KEY_CREATION_DATE = date(2013, 5, 14) + + # '0' is the magic value that tells GPG's batch key generation not + # to set an expiration date. + DEFAULT_KEY_EXPIRATION_DATE = "0" + + REDIS_FINGERPRINT_HASH = "sd/crypto-util/fingerprints" + REDIS_KEY_HASH = "sd/crypto-util/keys" + + SOURCE_KEY_NAME = "Source Key" + SOURCE_KEY_UID_RE = re.compile(r"(Source|Autogenerated) Key <[-A-Za-z0-9+/=_]+>") + + def __init__(self, gpg_key_dir: Path, journalist_key_fingerprint: str) -> None: + self._gpg_key_dir = gpg_key_dir + self._journalist_key_fingerprint = journalist_key_fingerprint + self._redis = Redis(decode_responses=True) + + # Instantiate the "main" GPG binary + gpg = gnupg.GPG(binary="gpg2", homedir=str(self._gpg_key_dir)) + if StrictVersion(gpg.binary_version) >= StrictVersion("2.1"): + # --pinentry-mode, required for SecureDrop on GPG 2.1.x+, was added in GPG 2.1. + self._gpg = gnupg.GPG( + binary="gpg2", homedir=str(gpg_key_dir), options=["--pinentry-mode loopback"] + ) + else: + self._gpg = gpg + + # Instantiate the GPG binary to be used for key deletion: always delete keys without + # invoking pinentry-mode=loopback + # see: https://lists.gnupg.org/pipermail/gnupg-users/2016-May/055965.html + self._gpg_for_key_deletion = gnupg.GPG( + binary="gpg2", homedir=str(self._gpg_key_dir), options=["--yes"] + ) + + # Ensure that the journalist public key has been previously imported in GPG + try: + self.get_journalist_public_key() + except GpgKeyNotFoundError: + raise EnvironmentError( + f"The journalist public key with fingerprint {journalist_key_fingerprint}" + f" has not been imported into GPG." + ) + + @classmethod + def get_default(cls) -> "EncryptionManager": + # Late import so the module can be used without a config.py in the parent folder + from sdconfig import config + + global _default_encryption_mgr + if _default_encryption_mgr is None: + _default_encryption_mgr = cls( + gpg_key_dir=Path(config.GPG_KEY_DIR), + journalist_key_fingerprint=config.JOURNALIST_KEY, + ) + return _default_encryption_mgr + + def generate_source_key_pair(self, source_user: "SourceUser") -> None: + gen_key_input = self._gpg.gen_key_input( + passphrase=source_user.gpg_secret, + name_email=source_user.filesystem_id, + key_type=self.GPG_KEY_TYPE, + key_length=self.GPG_KEY_LENGTH, + name_real=self.SOURCE_KEY_NAME, + creation_date=self.DEFAULT_KEY_CREATION_DATE.isoformat(), + expire_date=self.DEFAULT_KEY_EXPIRATION_DATE, + ) + new_key = self._gpg.gen_key(gen_key_input) + + # Store the newly-created key's fingerprint in Redis for faster lookups + self._save_key_fingerprint_to_redis(source_user.filesystem_id, str(new_key)) + + def delete_source_key_pair(self, source_filesystem_id: str) -> None: + source_key_fingerprint = self.get_source_key_fingerprint(source_filesystem_id) + + # The subkeys keyword argument deletes both secret and public keys + self._gpg_for_key_deletion.delete_keys(source_key_fingerprint, secret=True, subkeys=True) + + self._redis.hdel(self.REDIS_KEY_HASH, source_key_fingerprint) + self._redis.hdel(self.REDIS_FINGERPRINT_HASH, source_filesystem_id) + + def get_journalist_public_key(self) -> str: + return self._get_public_key(self._journalist_key_fingerprint) + + def get_source_public_key(self, source_filesystem_id: str) -> str: + source_key_fingerprint = self.get_source_key_fingerprint(source_filesystem_id) + return self._get_public_key(source_key_fingerprint) + + def get_source_key_fingerprint(self, source_filesystem_id: str) -> str: + source_key_fingerprint = self._redis.hget(self.REDIS_FINGERPRINT_HASH, source_filesystem_id) + if source_key_fingerprint: + return source_key_fingerprint + + # If the fingerprint was not in Redis, get it directly from GPG + source_key_details = self._get_source_key_details(source_filesystem_id) + source_key_fingerprint = source_key_details["fingerprint"] + self._save_key_fingerprint_to_redis(source_filesystem_id, source_key_fingerprint) + return source_key_fingerprint + + def encrypt_source_message(self, message_in: str, encrypted_message_path_out: Path) -> None: + message_as_stream = StringIO(message_in) + self._encrypt( + # A submission is only encrypted for the journalist key + using_keys_with_fingerprints=[self._journalist_key_fingerprint], + plaintext_in=message_as_stream, + ciphertext_path_out=encrypted_message_path_out, + ) + + def encrypt_source_file(self, file_in: typing.IO, encrypted_file_path_out: Path) -> None: + self._encrypt( + # A submission is only encrypted for the journalist key + using_keys_with_fingerprints=[self._journalist_key_fingerprint], + plaintext_in=file_in, + ciphertext_path_out=encrypted_file_path_out, + ) + + def encrypt_journalist_reply( + self, for_source_with_filesystem_id: str, reply_in: str, encrypted_reply_path_out: Path + ) -> None: + source_key_fingerprint = self.get_source_key_fingerprint(for_source_with_filesystem_id) + reply_as_stream = StringIO(reply_in) + self._encrypt( + # A reply is encrypted for both the journalist key and the source key + using_keys_with_fingerprints=[source_key_fingerprint, self._journalist_key_fingerprint], + plaintext_in=reply_as_stream, + ciphertext_path_out=encrypted_reply_path_out, + ) + + def decrypt_journalist_reply(self, for_source_user: "SourceUser", ciphertext_in: bytes) -> str: + ciphertext_as_stream = BytesIO(ciphertext_in) + out = self._gpg.decrypt_file(ciphertext_as_stream, passphrase=for_source_user.gpg_secret) + if not out.ok: + raise GpgDecryptError(out.stderr) + + return out.data.decode("utf-8") + + def _encrypt( + self, + using_keys_with_fingerprints: List[str], + plaintext_in: typing.IO, + ciphertext_path_out: Path, + ) -> None: + # Remove any spaces from provided fingerprints GPG outputs fingerprints + # with spaces for readability, but requires the spaces to be removed + # when using fingerprints to specify recipients. + sanitized_key_fingerprints = [fpr.replace(" ", "") for fpr in using_keys_with_fingerprints] + + out = self._gpg.encrypt( + plaintext_in, + *sanitized_key_fingerprints, + output=str(ciphertext_path_out), + always_trust=True, + armor=False, + ) + if not out.ok: + raise GpgEncryptError(out.stderr) + + def _get_source_key_details(self, source_filesystem_id: str) -> Dict[str, str]: + for key in self._gpg.list_keys(): + for uid in key["uids"]: + if source_filesystem_id in uid and self.SOURCE_KEY_UID_RE.match(uid): + return key + raise GpgKeyNotFoundError() + + def _save_key_fingerprint_to_redis( + self, source_filesystem_id: str, source_key_fingerprint: str + ) -> None: + self._redis.hset(self.REDIS_FINGERPRINT_HASH, source_filesystem_id, source_key_fingerprint) + + def _get_public_key(self, key_fingerprint: str) -> str: + # First try to fetch the public key from Redis + public_key = self._redis.hget(self.REDIS_KEY_HASH, key_fingerprint) + if public_key: + return public_key + + # Then directly from GPG + public_key = self._gpg.export_keys(key_fingerprint) + if not public_key: + raise GpgKeyNotFoundError() + + self._redis.hset(self.REDIS_KEY_HASH, key_fingerprint, public_key) + return public_key diff --git a/securedrop/journalist.py b/securedrop/journalist.py index c9f450005c..71c250275f 100644 --- a/securedrop/journalist.py +++ b/securedrop/journalist.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +from encryption import EncryptionManager, GpgKeyNotFoundError from sdconfig import config from journalist_app import create_app @@ -11,12 +11,14 @@ @asynchronous def prime_keycache() -> None: - """ - Preloads CryptoUtil.keycache. - """ + """Pre-load the source public keys into Redis.""" with app.app_context(): + encryption_mgr = EncryptionManager.get_default() for source in Source.query.filter_by(pending=False, deleted_at=None).all(): - app.crypto_util.get_pubkey(source.filesystem_id) + try: + encryption_mgr.get_source_public_key(source.filesystem_id) + except GpgKeyNotFoundError: + pass prime_keycache() diff --git a/securedrop/journalist_app/__init__.py b/securedrop/journalist_app/__init__.py index 83e87a11aa..0788cef590 100644 --- a/securedrop/journalist_app/__init__.py +++ b/securedrop/journalist_app/__init__.py @@ -15,7 +15,6 @@ import template_filters import version -from crypto_util import CryptoUtil from db import db from journalist_app import account, admin, api, main, col from journalist_app.utils import (get_source, logged_in, @@ -72,16 +71,7 @@ def create_app(config: 'SDConfig') -> Flask: # TODO: Attaching a Storage dynamically like this disables all type checking (and # breaks code analysis tools) for code that uses current_app.storage; it should be refactored - app.storage = Storage(config.STORE_DIR, - config.TEMP_DIR, - config.JOURNALIST_KEY) - - # TODO: Attaching a CryptoUtil dynamically like this disables all type checking (and - # breaks code analysis tools) for code that uses current_app.storage; it should be refactored - app.crypto_util = CryptoUtil( - securedrop_root=config.SECUREDROP_ROOT, - gpg_key_dir=config.GPG_KEY_DIR, - ) + app.storage = Storage(config.STORE_DIR, config.TEMP_DIR) @app.errorhandler(CSRFError) def handle_csrf_error(e: CSRFError) -> 'Response': diff --git a/securedrop/journalist_app/col.py b/securedrop/journalist_app/col.py index e88d764b09..1f0df2bd42 100644 --- a/securedrop/journalist_app/col.py +++ b/securedrop/journalist_app/col.py @@ -21,6 +21,7 @@ from sqlalchemy.orm.exc import NoResultFound from db import db +from encryption import GpgKeyNotFoundError, EncryptionManager from models import Reply, Submission from journalist_app.forms import ReplyForm from journalist_app.utils import (make_star_true, make_star_false, get_source, @@ -49,7 +50,12 @@ def remove_star(filesystem_id: str) -> werkzeug.Response: def col(filesystem_id: str) -> str: form = ReplyForm() source = get_source(filesystem_id) - source.has_key = current_app.crypto_util.get_fingerprint(filesystem_id) + try: + EncryptionManager.get_default().get_source_public_key(filesystem_id) + source.has_key = True + except GpgKeyNotFoundError: + source.has_key = False + return render_template("col.html", filesystem_id=filesystem_id, source=source, form=form) @@ -59,7 +65,7 @@ def delete_single(filesystem_id: str) -> werkzeug.Response: source = get_source(filesystem_id) try: delete_collection(filesystem_id) - except ValueError as e: + except GpgKeyNotFoundError as e: current_app.logger.error("error deleting collection: %s", e) abort(500) diff --git a/securedrop/journalist_app/main.py b/securedrop/journalist_app/main.py index 60151f400a..535e7b98c0 100644 --- a/securedrop/journalist_app/main.py +++ b/securedrop/journalist_app/main.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from datetime import datetime +from pathlib import Path from typing import Union import werkzeug @@ -12,6 +13,7 @@ import store from db import db +from encryption import EncryptionManager from models import SeenReply, Source, SourceStar, Submission, Reply from journalist_app.forms import ReplyForm from journalist_app.utils import (validate_user, bulk_delete, download, @@ -130,11 +132,10 @@ def reply() -> werkzeug.Response: g.source.interaction_count += 1 filename = "{0}-{1}-reply.gpg".format(g.source.interaction_count, g.source.journalist_filename) - current_app.crypto_util.encrypt( - form.message.data, - [current_app.crypto_util.get_fingerprint(g.filesystem_id), - config.JOURNALIST_KEY], - output=current_app.storage.path(g.filesystem_id, filename), + EncryptionManager.get_default().encrypt_journalist_reply( + for_source_with_filesystem_id=g.filesystem_id, + reply_in=form.message.data, + encrypted_reply_path_out=Path(current_app.storage.path(g.filesystem_id, filename)), ) try: diff --git a/securedrop/journalist_app/utils.py b/securedrop/journalist_app/utils.py index 28d5f221bc..172bbd439b 100644 --- a/securedrop/journalist_app/utils.py +++ b/securedrop/journalist_app/utils.py @@ -12,6 +12,7 @@ from sqlalchemy.exc import IntegrityError from db import db +from encryption import EncryptionManager from models import ( BadTokenException, FirstOrLastNameError, @@ -385,11 +386,7 @@ def delete_collection(filesystem_id: str) -> None: current_app.storage.move_to_shredder(path) # Delete the source's reply keypair - try: - current_app.crypto_util.delete_reply_keypair(filesystem_id) - except ValueError as e: - current_app.logger.error("could not delete reply keypair: %s", e) - raise + EncryptionManager.get_default().delete_source_key_pair(filesystem_id) # Delete their entry in the db source = get_source(filesystem_id, include_deleted=True) diff --git a/securedrop/loaddata.py b/securedrop/loaddata.py index 2f1f615f1b..9020b78954 100755 --- a/securedrop/loaddata.py +++ b/securedrop/loaddata.py @@ -8,6 +8,8 @@ import argparse import datetime import io +from pathlib import Path + import math import os import random @@ -20,6 +22,7 @@ import journalist_app from db import db +from encryption import EncryptionManager from models import ( Journalist, JournalistLoginAttempt, @@ -223,15 +226,11 @@ def add_reply( """ record_source_interaction(source) fname = "{}-{}-reply.gpg".format(source.interaction_count, source.journalist_filename) - current_app.crypto_util.encrypt( - next(replies), - [ - current_app.crypto_util.get_fingerprint(source.filesystem_id), - config.JOURNALIST_KEY, - ], - current_app.storage.path(source.filesystem_id, fname), + EncryptionManager.get_default().encrypt_journalist_reply( + for_source_with_filesystem_id=source.filesystem_id, + reply_in=next(replies), + encrypted_reply_path_out=Path(current_app.storage.path(source.filesystem_id, fname)), ) - reply = Reply(journalist, source, fname) db.session.add(reply) db.session.flush() @@ -262,7 +261,7 @@ def add_source() -> Tuple[Source, str]: db.session.commit() # Generate source key - current_app.crypto_util.genkeypair(source_user) + EncryptionManager.get_default().generate_source_key_pair(source_user) return source, codename diff --git a/securedrop/models.py b/securedrop/models.py index 82451409ca..3a664bb3eb 100644 --- a/securedrop/models.py +++ b/securedrop/models.py @@ -29,6 +29,7 @@ from logging import Logger from pyotp import TOTP, HOTP +from encryption import EncryptionManager, GpgKeyNotFoundError LOGIN_HARDENING = True if os.environ.get('SECUREDROP_ENV') == 'test': @@ -110,11 +111,17 @@ def collection(self) -> 'List[Union[Submission, Reply]]': @property def fingerprint(self) -> 'Optional[str]': - return current_app.crypto_util.get_fingerprint(self.filesystem_id) + try: + return EncryptionManager.get_default().get_source_key_fingerprint(self.filesystem_id) + except GpgKeyNotFoundError: + return None @property def public_key(self) -> 'Optional[str]': - return current_app.crypto_util.get_pubkey(self.filesystem_id) + try: + return EncryptionManager.get_default().get_source_public_key(self.filesystem_id) + except GpgKeyNotFoundError: + return None def to_json(self) -> 'Dict[str, Union[str, bool, int, str]]': docs_msg_count = self.documents_messages_count() diff --git a/securedrop/source_app/__init__.py b/securedrop/source_app/__init__.py index b168bc58c9..4ccc4ba660 100644 --- a/securedrop/source_app/__init__.py +++ b/securedrop/source_app/__init__.py @@ -15,7 +15,6 @@ import template_filters import version -from crypto_util import CryptoUtil from db import db from models import InstanceConfig from request_that_secures_file_uploads import RequestThatSecuresFileUploads @@ -68,16 +67,7 @@ def setup_i18n() -> None: # TODO: Attaching a Storage dynamically like this disables all type checking (and # breaks code analysis tools) for code that uses current_app.storage; it should be refactored - app.storage = Storage(config.STORE_DIR, - config.TEMP_DIR, - config.JOURNALIST_KEY) - - # TODO: Attaching a CryptoUtil dynamically like this disables all type checking (and - # breaks code analysis tools) for code that uses current_app.storage; it should be refactored - app.crypto_util = CryptoUtil( - securedrop_root=config.SECUREDROP_ROOT, - gpg_key_dir=config.GPG_KEY_DIR, - ) + app.storage = Storage(config.STORE_DIR, config.TEMP_DIR) @app.errorhandler(CSRFError) def handle_csrf_error(e: CSRFError) -> werkzeug.Response: diff --git a/securedrop/source_app/info.py b/securedrop/source_app/info.py index f626a3ecc3..b8c8285d0d 100644 --- a/securedrop/source_app/info.py +++ b/securedrop/source_app/info.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- import flask -from flask import Blueprint, render_template, send_file, current_app, redirect, url_for +from flask import Blueprint, render_template, send_file, redirect, url_for import werkzeug from io import BytesIO # noqa +from encryption import EncryptionManager from sdconfig import SDConfig @@ -21,8 +22,7 @@ def recommend_tor_browser() -> str: @view.route('/public-key') def download_public_key() -> flask.Response: - journalist_pubkey = current_app.crypto_util.gpg.export_keys( - config.JOURNALIST_KEY) + journalist_pubkey = EncryptionManager.get_default().get_journalist_public_key() data = BytesIO(journalist_pubkey.encode('utf-8')) return send_file(data, mimetype="application/pgp-keys", diff --git a/securedrop/source_app/main.py b/securedrop/source_app/main.py index e87d82e86e..e358f4f75e 100644 --- a/securedrop/source_app/main.py +++ b/securedrop/source_app/main.py @@ -14,6 +14,7 @@ import store from db import db +from encryption import EncryptionManager, GpgKeyNotFoundError from models import Submission, Reply, get_one_or_else from passphrases import PassphraseGenerator @@ -117,8 +118,11 @@ def lookup(logged_in_source: SourceUser) -> str: try: with io.open(reply_path, "rb") as f: contents = f.read() - reply_obj = current_app.crypto_util.decrypt(logged_in_source, contents) - reply.decrypted = reply_obj + decrypted_reply = EncryptionManager.get_default().decrypt_journalist_reply( + for_source_user=logged_in_source, + ciphertext_in=contents + ) + reply.decrypted = decrypted_reply except UnicodeDecodeError: current_app.logger.error("Could not decode reply %s" % reply.filename) @@ -133,9 +137,12 @@ def lookup(logged_in_source: SourceUser) -> str: # Sort the replies by date replies.sort(key=operator.attrgetter('date'), reverse=True) - # Generate a keypair to encrypt replies from the journalist - if not current_app.crypto_util.get_fingerprint(logged_in_source.filesystem_id): - current_app.crypto_util.genkeypair(logged_in_source) + # If not done yet, generate a keypair to encrypt replies from the journalist + encryption_mgr = EncryptionManager.get_default() + try: + encryption_mgr.get_source_public_key(logged_in_source.filesystem_id) + except GpgKeyNotFoundError: + encryption_mgr.generate_source_key_pair(logged_in_source) return render_template( 'lookup.html', diff --git a/securedrop/store.py b/securedrop/store.py index 12a8fff79d..a4792934c9 100644 --- a/securedrop/store.py +++ b/securedrop/store.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from pathlib import Path + import binascii import gzip import os @@ -12,6 +14,7 @@ from sqlalchemy.orm import sessionmaker from werkzeug.utils import secure_filename +from encryption import EncryptionManager from secure_tempfile import SecureTemporaryFile import rm @@ -89,7 +92,7 @@ def safe_renames(old: str, new: str) -> None: class Storage: - def __init__(self, storage_path: str, temp_dir: str, gpg_key: str) -> None: + def __init__(self, storage_path: str, temp_dir: str) -> None: if not os.path.isabs(storage_path): raise PathException("storage_path {} is not absolute".format( storage_path)) @@ -100,13 +103,15 @@ def __init__(self, storage_path: str, temp_dir: str, gpg_key: str) -> None: temp_dir)) self.__temp_dir = temp_dir - self.__gpg_key = gpg_key - # where files and directories are sent to be securely deleted self.__shredder_path = os.path.abspath(os.path.join(self.__storage_path, "../shredder")) if not os.path.exists(self.__shredder_path): os.makedirs(self.__shredder_path, mode=0o700) + # crash if we don't have a way to securely remove files + if not rm.check_secure_delete_capability(): + raise AssertionError("Secure file deletion is not possible.") + @property def storage_path(self) -> str: return self.__storage_path @@ -341,8 +346,10 @@ def save_file_submission(self, break gzf.write(buf) - current_app.crypto_util.encrypt( - stf, [self.__gpg_key], encrypted_file_path) + EncryptionManager.get_default().encrypt_source_file( + file_in=stf, + encrypted_file_path_out=Path(encrypted_file_path), + ) return encrypted_file_name @@ -370,7 +377,10 @@ def save_message_submission(self, message: str) -> str: filename = "{0}-{1}-msg.gpg".format(count, journalist_filename) msg_loc = self.path(filesystem_id, filename) - current_app.crypto_util.encrypt(message, [self.__gpg_key], msg_loc) + EncryptionManager.get_default().encrypt_source_message( + message_in=message, + encrypted_message_path_out=Path(msg_loc), + ) return filename diff --git a/securedrop/tests/conftest.py b/securedrop/tests/conftest.py index a7273448cf..e8e8aa7b2b 100644 --- a/securedrop/tests/conftest.py +++ b/securedrop/tests/conftest.py @@ -7,6 +7,7 @@ from typing import Any, Generator, Tuple from unittest import mock +from unittest.mock import PropertyMock import pretty_bad_protocol as gnupg import logging @@ -26,6 +27,7 @@ from typing import Dict import sdconfig +from encryption import EncryptionManager from passphrases import PassphraseGenerator from source_user import _SourceScryptManager, create_source_user @@ -131,12 +133,12 @@ def setup_journalist_key_and_gpg_folder() -> Generator[Tuple[str, Path], None, N journalist_public_key = journalist_public_key_path.read_text() journalist_key_fingerprint = gpg.import_keys(journalist_public_key).fingerprints[0] - # TODO(AD): Don't import the journalist secret key; will be removed in my next PR - journalist_secret_key_path = Path(__file__).parent / "files" / "test_journalist_key.sec" - journalist_secret_key = journalist_secret_key_path.read_text() - gpg.import_keys(journalist_secret_key) + # Reduce source GPG key length to speed up tests at the expense of security + with mock.patch.object( + EncryptionManager, "GPG_KEY_LENGTH", PropertyMock(return_value=1024) + ): - yield journalist_key_fingerprint, tmp_gpg_dir + yield journalist_key_fingerprint, tmp_gpg_dir finally: shutil.rmtree(tmp_gpg_dir, ignore_errors=True) @@ -270,7 +272,7 @@ def test_source(journalist_app: Flask) -> Dict[str, Any]: source_passphrase=passphrase, source_app_storage=journalist_app.storage, ) - journalist_app.crypto_util.genkeypair(source_user) + EncryptionManager.get_default().generate_source_key_pair(source_user) source = source_user.get_db_record() return {'source_user': source_user, # TODO(AD): Eventually the next keys could be removed as they are in source_user diff --git a/securedrop/tests/functional/functional_test.py b/securedrop/tests/functional/functional_test.py index feb5e39c70..55860f876c 100644 --- a/securedrop/tests/functional/functional_test.py +++ b/securedrop/tests/functional/functional_test.py @@ -37,6 +37,7 @@ import source_app import tests.utils.env as env from db import db +from encryption import EncryptionManager from models import Journalist from source_user import _SourceScryptManager @@ -296,7 +297,7 @@ def wait_for_source_key(self, source_name): ) def key_available(filesystem_id): - assert self.source_app.crypto_util.get_fingerprint(filesystem_id) + assert EncryptionManager.get_default().get_source_key_fingerprint(filesystem_id) self.wait_for(lambda: key_available(filesystem_id), timeout=60) diff --git a/securedrop/tests/functional/journalist_navigation_steps.py b/securedrop/tests/functional/journalist_navigation_steps.py index cac28a5a49..5f9294ba9c 100644 --- a/securedrop/tests/functional/journalist_navigation_steps.py +++ b/securedrop/tests/functional/journalist_navigation_steps.py @@ -21,7 +21,9 @@ # Number of times to try flaky clicks. +from encryption import EncryptionManager from source_user import _SourceScryptManager +from tests.test_encryption import import_journalist_private_key CLICK_ATTEMPTS = 15 @@ -826,7 +828,9 @@ def cookie_string_from_selenium_cookies(cookies): cks = cookie_string_from_selenium_cookies(self.driver.get_cookies()) raw_content = self.return_downloaded_content(file_url, cks) - decrypted_submission = self.journalist_app.crypto_util.gpg.decrypt(raw_content) + encryption_mgr = EncryptionManager.get_default() + with import_journalist_private_key(encryption_mgr): + decrypted_submission = encryption_mgr._gpg.decrypt(raw_content) submission = self._get_submission_content(file_url, decrypted_submission) if type(submission) == bytes: submission = submission.decode("utf-8") @@ -1049,7 +1053,7 @@ def _source_delete_key(self): filesystem_id = _SourceScryptManager.get_default().derive_source_filesystem_id( self.source_name ) - self.source_app.crypto_util.delete_reply_keypair(filesystem_id) + EncryptionManager.get_default().delete_source_key_pair(filesystem_id) def _journalist_continues_after_flagging(self): self.wait_for(lambda: self.driver.find_element_by_id("continue-to-list")) diff --git a/securedrop/tests/test_crypto_util.py b/securedrop/tests/test_crypto_util.py deleted file mode 100644 index a60e9f6aa5..0000000000 --- a/securedrop/tests/test_crypto_util.py +++ /dev/null @@ -1,259 +0,0 @@ -# -*- coding: utf-8 -*- -from datetime import datetime -from hypothesis import given -from hypothesis.strategies import text -import io -import os -import pytest - -from passphrases import PassphraseGenerator - -from source_user import create_source_user - -from crypto_util import CryptoUtil, CryptoException -from db import db - - -def test_encrypt_success(source_app, config, test_source): - message = 'test' - - with source_app.app_context(): - ciphertext = source_app.crypto_util.encrypt( - message, - [source_app.crypto_util.get_fingerprint(test_source['filesystem_id']), - config.JOURNALIST_KEY], - source_app.storage.path(test_source['filesystem_id'], - 'somefile.gpg')) - - assert isinstance(ciphertext, bytes) - assert ciphertext.decode('utf-8') != message - assert len(ciphertext) > 0 - - -def test_encrypt_failure(source_app, test_source): - with source_app.app_context(): - with pytest.raises(CryptoException) as err: - source_app.crypto_util.encrypt( - str(os.urandom(1)), - [], - source_app.storage.path(test_source['filesystem_id'], - 'other.gpg')) - assert 'no terminal at all requested' in str(err) - - -def test_encrypt_without_output(source_app, config, test_source): - """We simply do not specify the option output keyword argument - to crypto_util.encrypt() here in order to confirm encryption - works when it defaults to `None`. - """ - message = 'test' - with source_app.app_context(): - ciphertext = source_app.crypto_util.encrypt( - message, - [source_app.crypto_util.get_fingerprint(test_source['filesystem_id']), - config.JOURNALIST_KEY]) - source_user = test_source["source_user"] - plaintext = source_app.crypto_util.decrypt( - source_user, - ciphertext) - - assert plaintext == message - - -def test_encrypt_binary_stream(source_app, config, test_source): - """Generally, we pass unicode strings (the type form data is - returned as) as plaintext to crypto_util.encrypt(). These have - to be converted to "binary stream" types (such as `file`) before - we can actually call gnupg.GPG.encrypt() on them. This is done - in crypto_util.encrypt() with an `if` branch that uses - `gnupg._util._is_stream(plaintext)` as the predicate, and calls - `gnupg._util._make_binary_stream(plaintext)` if necessary. This - test ensures our encrypt function works even if we provide - inputs such that this `if` branch is skipped (i.e., the object - passed for `plaintext` is one such that - `gnupg._util._is_stream(plaintext)` returns `True`). - """ - with source_app.app_context(): - source_user = test_source["source_user"] - with io.open(os.path.realpath(__file__)) as fh: - ciphertext = source_app.crypto_util.encrypt( - fh, - [source_app.crypto_util.get_fingerprint(source_user.filesystem_id), - config.JOURNALIST_KEY], - source_app.storage.path(source_user.filesystem_id, 'somefile.gpg')) - - plaintext = source_app.crypto_util.decrypt(source_user, ciphertext) - - with io.open(os.path.realpath(__file__)) as fh: - assert fh.read() == plaintext - - -def test_basic_encrypt_then_decrypt_multiple_recipients(source_app, - config, - test_source): - message = 'test' - - with source_app.app_context(): - source_user = test_source["source_user"] - ciphertext = source_app.crypto_util.encrypt( - message, - [source_app.crypto_util.get_fingerprint(source_user.filesystem_id), - config.JOURNALIST_KEY], - source_app.storage.path(source_user.filesystem_id, 'somefile.gpg')) - plaintext = source_app.crypto_util.decrypt(source_user, ciphertext) - - assert plaintext == message - - # Since there's no way to specify which key to use for - # decryption to python-gnupg, we delete the `source`'s key and - # ensure we can decrypt with the `config.JOURNALIST_KEY`. - source_app.crypto_util.delete_reply_keypair(source_user.filesystem_id) - plaintext = source_app.crypto_util.gpg.decrypt(ciphertext).data.decode('utf-8') - - assert plaintext == message - - -def test_genkeypair(source_app): - with source_app.app_context(): - source_user = create_source_user( - db_session=db.session, - source_passphrase=PassphraseGenerator.get_default().generate_passphrase(), - source_app_storage=source_app.storage, - ) - source_app.crypto_util.genkeypair(source_user) - - assert source_app.crypto_util.get_fingerprint(source_user.filesystem_id) is not None - - -def parse_gpg_date_string(date_string): - """Parse a date string returned from `gpg --with-colons --list-keys` into a - datetime. - - The format of the date strings is complicated; see gnupg doc/DETAILS for a - full explanation. - - Key details: - - The creation date of the key is given in UTC. - - the date is usually printed in seconds since epoch, however, we are - migrating to an ISO 8601 format (e.g. "19660205T091500"). A simple - way to detect the new format is to scan for the 'T'. - """ - if 'T' in date_string: - dt = datetime.strptime(date_string, "%Y%m%dT%H%M%S") - else: - dt = datetime.utcfromtimestamp(int(date_string)) - return dt - - -def test_reply_keypair_creation_and_expiration_dates(source_app): - with source_app.app_context(): - source_user = create_source_user( - db_session=db.session, - source_passphrase=PassphraseGenerator.get_default().generate_passphrase(), - source_app_storage=source_app.storage, - ) - source_app.crypto_util.genkeypair(source_user) - - # crypto_util.get_fingerprint only returns the fingerprint of the key. We need - # the full output of gpg.list_keys() to check the creation and - # expire dates. - # - # TODO: it might be generally useful to refactor crypto_util.get_fingerprint so - # it always returns the entire key dictionary instead of just the - # fingerprint (which is always easily extracted from the entire key - # dictionary). - new_key_fingerprint = source_app.crypto_util.get_fingerprint(source_user.filesystem_id) - new_key = [key for key in source_app.crypto_util.gpg.list_keys() - if new_key_fingerprint == key['fingerprint']][0] - - # All keys should share the same creation date to avoid leaking - # information about when sources first created accounts. - creation_date = parse_gpg_date_string(new_key['date']) - assert (creation_date.date() == - CryptoUtil.DEFAULT_KEY_CREATION_DATE) - - # Reply keypairs should not expire - expire_date = new_key['expires'] - assert expire_date == '' - - -def test_delete_reply_keypair(source_app, test_source): - fid = test_source['filesystem_id'] - source_app.crypto_util.delete_reply_keypair(fid) - assert source_app.crypto_util.get_fingerprint(fid) is None - - -def test_delete_reply_keypair_pinentry_status_is_handled(source_app, test_source, - mocker, capsys): - """ - Regression test for https://github.com/freedomofpress/securedrop/issues/4294 - """ - fid = test_source['filesystem_id'] - - # Patch private python-gnupg method to reproduce the issue in #4294 - mocker.patch('pretty_bad_protocol._util._separate_keyword', - return_value=('PINENTRY_LAUNCHED', 'does not matter')) - - source_app.crypto_util.delete_reply_keypair(fid) - - captured = capsys.readouterr() - assert "ValueError: Unknown status message: 'PINENTRY_LAUNCHED'" not in captured.err - assert source_app.crypto_util.get_fingerprint(fid) is None - - -def test_delete_reply_keypair_no_key(source_app): - """No exceptions should be raised when provided a filesystem id that - does not exist. - """ - source_app.crypto_util.delete_reply_keypair('Reality Winner') - - -def test_delete_reply_keypair_non_source(source_app): - """ - Checks that a non-source key is not deleted by delete_reply_keypair. - """ - name = "SecureDrop Test/Development (DO NOT USE IN PRODUCTION)" - with pytest.raises(ValueError) as excinfo: - source_app.crypto_util.delete_reply_keypair(name) - assert "source key not found" in str(excinfo.value) - assert source_app.crypto_util.get_fingerprint(name) - - -def test_get_fingerprint(source_app, test_source): - assert (source_app.crypto_util.get_fingerprint(test_source['filesystem_id']) - is not None) - - # check that a non-existent key returns None - assert source_app.crypto_util.get_fingerprint('x' * 50) is None - - -def test_get_pubkey(source_app, test_source): - begin_pgp = '-----BEGIN PGP PUBLIC KEY BLOCK----' - - # check that a filesystem_id exports the pubkey - pubkey = source_app.crypto_util.get_pubkey(test_source['filesystem_id']) - assert pubkey.startswith(begin_pgp) - - # check that a non-existent identifer exports None - pubkey = source_app.crypto_util.get_pubkey('x' * 50) - assert pubkey is None - - -@given(message=text()) -def test_encrypt_then_decrypt_gives_same_result( - source_app, - test_source, - message -): - """Test that encrypting, then decrypting a string gives the original string. - - This is the first test case using `hypothesis`: - https://hypothesis.readthedocs.io - """ - crypto = source_app.crypto_util - source_user = test_source["source_user"] - key = crypto.genkeypair(source_user) - ciphertext = crypto.encrypt(message, [str(key)]) - decrypted_text = crypto.decrypt(source_user, ciphertext) - - assert decrypted_text == message diff --git a/securedrop/tests/test_encryption.py b/securedrop/tests/test_encryption.py new file mode 100644 index 0000000000..7556d09290 --- /dev/null +++ b/securedrop/tests/test_encryption.py @@ -0,0 +1,351 @@ +import typing +from contextlib import contextmanager +from pathlib import Path + +import pytest as pytest + +from db import db +from encryption import EncryptionManager, GpgKeyNotFoundError, GpgEncryptError, GpgDecryptError +from datetime import datetime +from passphrases import PassphraseGenerator +from source_user import create_source_user + + +class TestEncryptionManager: + def test_get_default(self, config): + encryption_mgr = EncryptionManager.get_default() + assert encryption_mgr + assert encryption_mgr.get_journalist_public_key() + + def test_generate_source_key_pair(self, setup_journalist_key_and_gpg_folder, source_app): + # Given a source user + with source_app.app_context(): + source_user = create_source_user( + db_session=db.session, + source_passphrase=PassphraseGenerator.get_default().generate_passphrase(), + source_app_storage=source_app.storage, + ) + + # And an encryption manager + journalist_key_fingerprint, gpg_key_dir = setup_journalist_key_and_gpg_folder + encryption_mgr = EncryptionManager( + gpg_key_dir=gpg_key_dir, journalist_key_fingerprint=journalist_key_fingerprint + ) + + # When using the encryption manager to generate a key pair for this source user + # It succeeds + encryption_mgr.generate_source_key_pair(source_user) + + # And the newly-created key's fingerprint was added to Redis + fingerprint_in_redis = encryption_mgr._redis.hget( + encryption_mgr.REDIS_FINGERPRINT_HASH, source_user.filesystem_id + ) + assert fingerprint_in_redis + source_key_fingerprint = encryption_mgr.get_source_key_fingerprint( + source_user.filesystem_id + ) + assert fingerprint_in_redis == source_key_fingerprint + + # And the user's newly-generated public key can be retrieved + assert encryption_mgr.get_source_public_key(source_user.filesystem_id) + + # And the key has a hardcoded creation date to avoid leaking information about when sources + # first created their account + source_key_details = encryption_mgr._get_source_key_details(source_user.filesystem_id) + assert source_key_details + creation_date = _parse_gpg_date_string(source_key_details["date"]) + assert creation_date.date() == EncryptionManager.DEFAULT_KEY_CREATION_DATE + + # And the user's key does not expire + assert source_key_details["expires"] == "" + + def test_get_source_public_key(self, source_app, test_source): + # Given a source user with a key pair in the default encryption manager + source_user = test_source["source_user"] + encryption_mgr = EncryptionManager.get_default() + + # When using the encryption manager to fetch the source user's public key + # It succeeds + source_pub_key = encryption_mgr.get_source_public_key(source_user.filesystem_id) + assert source_pub_key + assert source_pub_key.startswith("-----BEGIN PGP PUBLIC KEY BLOCK----") + + # And the key's fingerprint was saved to Redis + source_key_fingerprint = encryption_mgr._redis.hget( + encryption_mgr.REDIS_FINGERPRINT_HASH, source_user.filesystem_id + ) + assert source_key_fingerprint + + # And the public key was saved to Redis + assert encryption_mgr._redis.hget(encryption_mgr.REDIS_KEY_HASH, source_key_fingerprint) + + def test_get_journalist_public_key(self, setup_journalist_key_and_gpg_folder): + # Given an encryption manager + journalist_key_fingerprint, gpg_key_dir = setup_journalist_key_and_gpg_folder + encryption_mgr = EncryptionManager( + gpg_key_dir=gpg_key_dir, journalist_key_fingerprint=journalist_key_fingerprint + ) + + # When using the encryption manager to fetch the journalist public key + # It succeeds + journalist_pub_key = encryption_mgr.get_journalist_public_key() + assert journalist_pub_key + assert journalist_pub_key.startswith("-----BEGIN PGP PUBLIC KEY BLOCK----") + + def test_get_source_public_key_wrong_id(self, setup_journalist_key_and_gpg_folder): + # Given an encryption manager + journalist_key_fingerprint, gpg_key_dir = setup_journalist_key_and_gpg_folder + encryption_mgr = EncryptionManager( + gpg_key_dir=gpg_key_dir, journalist_key_fingerprint=journalist_key_fingerprint + ) + + # When using the encryption manager to fetch a key for an invalid filesystem id + # It fails + with pytest.raises(GpgKeyNotFoundError): + encryption_mgr.get_source_public_key("1234test") + + def test_delete_source_key_pair(self, source_app, test_source): + # Given a source user with a key pair in the default encryption manager + source_user = test_source["source_user"] + encryption_mgr = EncryptionManager.get_default() + + # When using the encryption manager to delete this source user's key pair + # It succeeds + encryption_mgr.delete_source_key_pair(source_user.filesystem_id) + + # And the user's key information can no longer be retrieved + with pytest.raises(GpgKeyNotFoundError): + encryption_mgr.get_source_public_key(source_user.filesystem_id) + + with pytest.raises(GpgKeyNotFoundError): + encryption_mgr.get_source_key_fingerprint(source_user.filesystem_id) + + with pytest.raises(GpgKeyNotFoundError): + encryption_mgr._get_source_key_details(source_user.filesystem_id) + + def test_delete_source_key_pair_on_journalist_key(self, setup_journalist_key_and_gpg_folder): + # Given an encryption manager + journalist_key_fingerprint, gpg_key_dir = setup_journalist_key_and_gpg_folder + encryption_mgr = EncryptionManager( + gpg_key_dir=gpg_key_dir, journalist_key_fingerprint=journalist_key_fingerprint + ) + + # When trying to delete the journalist key via the encryption manager + # It fails + with pytest.raises(GpgKeyNotFoundError): + encryption_mgr.delete_source_key_pair(journalist_key_fingerprint) + + def test_delete_source_key_pair_pinentry_status_is_handled( + self, source_app, test_source, mocker, capsys + ): + """ + Regression test for https://github.com/freedomofpress/securedrop/issues/4294 + """ + # Given a source user with a key pair in the default encryption manager + source_user = test_source["source_user"] + encryption_mgr = EncryptionManager.get_default() + + # And a gpg binary that will trigger the issue described in #4294 + mocker.patch( + "pretty_bad_protocol._util._separate_keyword", + return_value=("PINENTRY_LAUNCHED", "does not matter"), + ) + + # When using the encryption manager to delete this source user's key pair + # It succeeds + encryption_mgr.delete_source_key_pair(source_user.filesystem_id) + + # And the user's key information can no longer be retrieved + with pytest.raises(GpgKeyNotFoundError): + encryption_mgr.get_source_key_fingerprint(source_user.filesystem_id) + + # And the bug fix was properly triggered + captured = capsys.readouterr() + assert "ValueError: Unknown status message: 'PINENTRY_LAUNCHED'" not in captured.err + + def test_encrypt_source_message(self, setup_journalist_key_and_gpg_folder, tmp_path): + # Given an encryption manager + journalist_key_fingerprint, gpg_key_dir = setup_journalist_key_and_gpg_folder + encryption_mgr = EncryptionManager( + gpg_key_dir=gpg_key_dir, journalist_key_fingerprint=journalist_key_fingerprint + ) + + # And a message to be submitted by a source + message = "s3cr3t message" + + # When the source tries to encrypt the message + # It succeeds + encrypted_message_path = tmp_path / "message.gpg" + encryption_mgr.encrypt_source_message( + message_in=message, encrypted_message_path_out=encrypted_message_path + ) + + # And the output file contains the encrypted data + encrypted_message = encrypted_message_path.read_bytes() + assert encrypted_message + + # And the journalist is able to decrypt the message + with import_journalist_private_key(encryption_mgr): + decrypted_message = encryption_mgr._gpg.decrypt(encrypted_message).data + assert decrypted_message.decode() == message + + # And the source or anyone else is NOT able to decrypt the message + # For GPG 2.1+, a non-null passphrase _must_ be passed to decrypt() + assert not encryption_mgr._gpg.decrypt(encrypted_message, passphrase="test 123").ok + + def test_encrypt_source_file(self, setup_journalist_key_and_gpg_folder, tmp_path): + # Given an encryption manager + journalist_key_fingerprint, gpg_key_dir = setup_journalist_key_and_gpg_folder + encryption_mgr = EncryptionManager( + gpg_key_dir=gpg_key_dir, journalist_key_fingerprint=journalist_key_fingerprint + ) + + # And a file to be submitted by a source - we use this python file + file_to_encrypt_path = Path(__file__) + with file_to_encrypt_path.open() as file_to_encrypt: + + # When the source tries to encrypt the file + # It succeeds + encrypted_file_path = tmp_path / "file.gpg" + encryption_mgr.encrypt_source_file( + file_in=file_to_encrypt, + encrypted_file_path_out=encrypted_file_path, + ) + + # And the output file contains the encrypted data + encrypted_file = encrypted_file_path.read_bytes() + assert encrypted_file + + # And the journalist is able to decrypt the file + with import_journalist_private_key(encryption_mgr): + decrypted_file = encryption_mgr._gpg.decrypt(encrypted_file).data + assert decrypted_file.decode() == file_to_encrypt_path.read_text() + + # And the source or anyone else is NOT able to decrypt the file + # For GPG 2.1+, a non-null passphrase _must_ be passed to decrypt() + assert not encryption_mgr._gpg.decrypt(encrypted_file, passphrase="test 123").ok + + def test_encrypt_and_decrypt_journalist_reply(self, source_app, test_source, tmp_path): + # Given a source user with a key pair in the default encryption manager + source_user1 = test_source["source_user"] + encryption_mgr = EncryptionManager.get_default() + + # And another source with a key pair in the default encryption manager + with source_app.app_context(): + source_user2 = create_source_user( + db_session=db.session, + source_passphrase=PassphraseGenerator.get_default().generate_passphrase(), + source_app_storage=source_app.storage, + ) + encryption_mgr.generate_source_key_pair(source_user2) + + # When the journalist tries to encrypt a reply to source1 + # It succeeds + journalist_reply = "s3cr3t message" + encrypted_reply_path = tmp_path / "reply.gpg" + encryption_mgr.encrypt_journalist_reply( + for_source_with_filesystem_id=source_user1.filesystem_id, + reply_in=journalist_reply, + encrypted_reply_path_out=encrypted_reply_path, + ) + + # And the output file contains the encrypted data + encrypted_reply = encrypted_reply_path.read_bytes() + assert encrypted_reply + + # And source1 is able to decrypt the reply + decrypted_reply = encryption_mgr.decrypt_journalist_reply( + for_source_user=source_user1, ciphertext_in=encrypted_reply + ) + assert decrypted_reply + assert decrypted_reply == journalist_reply + + # And source2 is NOT able to decrypt the reply + with pytest.raises(GpgDecryptError): + encryption_mgr.decrypt_journalist_reply( + for_source_user=source_user2, ciphertext_in=encrypted_reply + ) + + # Amd the reply can't be decrypted without providing the source1's gpg secret + result = encryption_mgr._gpg.decrypt( + # For GPG 2.1+, a non-null passphrase _must_ be passed to decrypt() + encrypted_reply, + passphrase="test 123", + ) + assert not result.ok + + # And the journalist is able to decrypt their reply + with import_journalist_private_key(encryption_mgr): + decrypted_reply_for_journalist = encryption_mgr._gpg.decrypt( + # For GPG 2.1+, a non-null passphrase _must_ be passed to decrypt() + encrypted_reply, + passphrase="test 123", + ).data + assert decrypted_reply_for_journalist.decode() == journalist_reply + + def test_encrypt_fails(self, setup_journalist_key_and_gpg_folder, tmp_path): + # Given an encryption manager + journalist_key_fingerprint, gpg_key_dir = setup_journalist_key_and_gpg_folder + encryption_mgr = EncryptionManager( + gpg_key_dir=gpg_key_dir, journalist_key_fingerprint=journalist_key_fingerprint + ) + + # When trying to encrypt some data without providing any recipient + # It fails and the right exception is raised + with pytest.raises(GpgEncryptError) as exc: + encryption_mgr._encrypt( + using_keys_with_fingerprints=[], + plaintext_in="test", + ciphertext_path_out=tmp_path / "encrypt_fails", + ) + assert "no terminal at all requested" in str(exc) + + +def _parse_gpg_date_string(date_string: str) -> datetime: + """Parse a date string returned from `gpg --with-colons --list-keys` into a datetime. + + The format of the date strings is complicated; see gnupg doc/DETAILS for a + full explanation. + + Key details: + - The creation date of the key is given in UTC. + - the date is usually printed in seconds since epoch, however, we are + migrating to an ISO 8601 format (e.g. "19660205T091500"). A simple + way to detect the new format is to scan for the 'T'. + """ + if "T" in date_string: + dt = datetime.strptime(date_string, "%Y%m%dT%H%M%S") + else: + dt = datetime.utcfromtimestamp(int(date_string)) + return dt + + +@contextmanager +def import_journalist_private_key( + encryption_mgr: EncryptionManager, +) -> typing.Generator[None, None, None]: + """Import the journalist secret key so the encryption_mgr can decrypt data for the journalist. + + The journalist secret key is removed at the end of this context manager in order to not impact + other decryption-related tests. + """ + # Import the journalist private key + journalist_private_key_path = Path(__file__).parent / "files" / "test_journalist_key.sec" + encryption_mgr._gpg.import_keys(journalist_private_key_path.read_text()) + journalist_secret_key_fingerprint = "C1C4E16BB24E4F4ABF37C3A6C3E7C4C0A2201B2A" + + yield + + # Make sure to remove the journalist private key to not impact the other tests + encryption_mgr._gpg_for_key_deletion.delete_keys( + fingerprints=journalist_secret_key_fingerprint, secret=True, subkeys=False + ) + + # Double check that the journlist private key was removed + is_journalist_secret_key_available = False + for key in encryption_mgr._gpg.list_keys(secret=True): + for uid in key["uids"]: + if "SecureDrop Test" in uid: + is_journalist_secret_key_available = True + break + assert not is_journalist_secret_key_available diff --git a/securedrop/tests/test_integration.py b/securedrop/tests/test_integration.py index 376de70c37..43a012aaf6 100644 --- a/securedrop/tests/test_integration.py +++ b/securedrop/tests/test_integration.py @@ -7,8 +7,8 @@ import zipfile from base64 import b32encode from binascii import unhexlify -from distutils.version import StrictVersion from io import BytesIO +import mock from bs4 import BeautifulSoup from flask import current_app, escape, g, session @@ -16,8 +16,10 @@ import journalist_app as journalist_app_module from db import db +from encryption import EncryptionManager from source_app.session_manager import SessionManager from . import utils +from .test_encryption import import_journalist_private_key from .utils.instrument import InstrumentedApp @@ -82,9 +84,12 @@ def test_submit_message(journalist_app, source_app, test_journo): resp = app.get(submission_url) assert resp.status_code == 200 - decrypted_data = journalist_app.crypto_util.gpg.decrypt(resp.data) - assert decrypted_data.ok - assert decrypted_data.data.decode('utf-8') == test_msg + + encryption_mgr = EncryptionManager.get_default() + with import_journalist_private_key(encryption_mgr): + decryption_result = encryption_mgr._gpg.decrypt(resp.data) + assert decryption_result.ok + assert decryption_result.data.decode('utf-8') == test_msg # delete submission resp = app.get(col_url) @@ -181,7 +186,10 @@ def test_submit_file(journalist_app, source_app, test_journo): resp = app.get(submission_url) assert resp.status_code == 200 - decrypted_data = journalist_app.crypto_util.gpg.decrypt(resp.data) + + encryption_mgr = EncryptionManager.get_default() + with import_journalist_private_key(encryption_mgr): + decrypted_data = encryption_mgr._gpg.decrypt(resp.data) assert decrypted_data.ok sio = BytesIO(decrypted_data.data) @@ -270,7 +278,7 @@ def _helper_test_reply(journalist_app, source_app, config, test_journo, resp = app.get(col_url) assert resp.status_code == 200 - assert current_app.crypto_util.get_fingerprint(filesystem_id) is not None + assert EncryptionManager.get_default().get_source_key_fingerprint(filesystem_id) # Create 2 replies to test deleting on journalist and source interface with journalist_app.test_client() as app: @@ -308,11 +316,8 @@ def _helper_test_reply(journalist_app, source_app, config, test_journo, zf = zipfile.ZipFile(BytesIO(resp.data), 'r') data = zf.read(zf.namelist()[0]) - _can_decrypt_with_key(journalist_app, data) - _can_decrypt_with_key( - journalist_app, - data, - source_user.gpg_secret) + _can_decrypt_with_journalist_secret_key(data) + _can_decrypt_with_source_secret_key(data, source_user.gpg_secret) # Test deleting reply on the journalist interface last_reply_number = len( @@ -381,27 +386,24 @@ def assertion(): utils.asynchronous.wait_for_assertion(assertion) -def _can_decrypt_with_key(journalist_app, msg, source_gpg_secret=None): - """ - Test that the given GPG message can be decrypted. - """ +def _can_decrypt_with_journalist_secret_key(msg: bytes) -> None: + encryption_mgr = EncryptionManager.get_default() + with import_journalist_private_key(encryption_mgr): + # For GPG 2.1+, a non null passphrase _must_ be passed to decrypt() + decryption_result = encryption_mgr._gpg.decrypt(msg, passphrase="dummy passphrase") + + assert decryption_result.ok, \ + "Could not decrypt msg with key, gpg says: {}" \ + .format(decryption_result.stderr) - # For GPG 2.1+, a non null passphrase _must_ be passed to decrypt() - using_gpg_2_1 = StrictVersion( - journalist_app.crypto_util.gpg.binary_version) >= StrictVersion('2.1') - if source_gpg_secret: - final_passphrase = source_gpg_secret - elif using_gpg_2_1: - final_passphrase = 'dummy passphrase' - else: - final_passphrase = None +def _can_decrypt_with_source_secret_key(msg: bytes, source_gpg_secret: str) -> None: + encryption_mgr = EncryptionManager.get_default() + decryption_result = encryption_mgr._gpg.decrypt(msg, passphrase=source_gpg_secret) - decrypted_data = journalist_app.crypto_util.gpg.decrypt( - msg, passphrase=final_passphrase) - assert decrypted_data.ok, \ + assert decryption_result.ok, \ "Could not decrypt msg with key, gpg says: {}" \ - .format(decrypted_data.stderr) + .format(decryption_result.stderr) def test_reply_normal(journalist_app, @@ -411,10 +413,11 @@ def test_reply_normal(journalist_app, '''Test for regression on #1360 (failure to encode bytes before calling gpg functions). ''' - journalist_app.crypto_util.gpg._encoding = "ansi_x3.4_1968" - source_app.crypto_util.gpg._encoding = "ansi_x3.4_1968" - _helper_test_reply(journalist_app, source_app, config, test_journo, - "This is a test reply.", True) + encryption_mgr = EncryptionManager.get_default() + with mock.patch.object(encryption_mgr._gpg, "_encoding", "ansi_x3.4_1968"): + _helper_test_reply( + journalist_app, source_app, config, test_journo, "This is a test reply.", True + ) def test_unicode_reply_with_ansi_env(journalist_app, @@ -429,10 +432,11 @@ def test_unicode_reply_with_ansi_env(journalist_app, # _encoding attribute it would have had it been initialized in a "C" # environment. See # https://github.com/freedomofpress/securedrop/issues/1360 for context. - journalist_app.crypto_util.gpg._encoding = "ansi_x3.4_1968" - source_app.crypto_util.gpg._encoding = "ansi_x3.4_1968" - _helper_test_reply(journalist_app, source_app, config, test_journo, - "ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ", True) + encryption_mgr = EncryptionManager.get_default() + with mock.patch.object(encryption_mgr._gpg, "_encoding", "ansi_x3.4_1968"): + _helper_test_reply( + journalist_app, source_app, config, test_journo, "ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ", True + ) def test_delete_collection(mocker, source_app, journalist_app, test_journo): diff --git a/securedrop/tests/test_journalist.py b/securedrop/tests/test_journalist.py index 1f3b3bbbd1..bd6549184b 100644 --- a/securedrop/tests/test_journalist.py +++ b/securedrop/tests/test_journalist.py @@ -23,6 +23,7 @@ from html import escape as htmlescape import journalist_app as journalist_app_module +from encryption import EncryptionManager, GpgKeyNotFoundError from journalist_app.utils import mark_seen import models from db import db @@ -2489,17 +2490,15 @@ def test_delete_source_deletes_source_key(journalist_app, utils.db_helper.reply(journo, source, 2) # Source key exists - source_key = current_app.crypto_util.get_fingerprint( - test_source['filesystem_id']) - assert source_key is not None + encryption_mgr = EncryptionManager.get_default() + assert encryption_mgr.get_source_key_fingerprint(test_source['filesystem_id']) journalist_app_module.utils.delete_collection( test_source['filesystem_id']) # Source key no longer exists - source_key = current_app.crypto_util.get_fingerprint( - test_source['filesystem_id']) - assert source_key is None + with pytest.raises(GpgKeyNotFoundError): + encryption_mgr.get_source_key_fingerprint(test_source['filesystem_id']) def test_delete_source_deletes_docs_on_disk(journalist_app, diff --git a/securedrop/tests/test_journalist_api.py b/securedrop/tests/test_journalist_api.py index 8bae9207e8..0fc2f44d2b 100644 --- a/securedrop/tests/test_journalist_api.py +++ b/securedrop/tests/test_journalist_api.py @@ -8,6 +8,7 @@ from uuid import UUID, uuid4 from db import db +from encryption import EncryptionManager from models import ( Journalist, Reply, @@ -724,9 +725,11 @@ def test_authorized_user_can_add_reply(journalist_app, journalist_api_token, # First we must encrypt the reply, or it will get rejected # by the server. - source_key = journalist_app.crypto_util.get_fingerprint( - test_source['source'].filesystem_id) - reply_content = journalist_app.crypto_util.gpg.encrypt( + encryption_mgr = EncryptionManager.get_default() + source_key = encryption_mgr.get_source_key_fingerprint( + test_source['source'].filesystem_id + ) + reply_content = encryption_mgr._gpg.encrypt( 'This is a plaintext reply', source_key).data response = app.post(url_for('api.all_source_replies', diff --git a/securedrop/tests/test_manage.py b/securedrop/tests/test_manage.py index 3ccd0efdd5..c7f805d0e8 100644 --- a/securedrop/tests/test_manage.py +++ b/securedrop/tests/test_manage.py @@ -100,7 +100,7 @@ def test_exception_handling_when_duplicate_username(journalist_app, assert 'successfully added' in out # Inserting the user for a second time should fail - return_value = manage._add_user() + return_value = manage._add_user(context=context) out, err = capsys.readouterr() assert return_value == 1 assert 'ERROR: That username is already taken!' in out @@ -120,7 +120,7 @@ def test_delete_user(journalist_app, config, mocker): return_value = manage._add_user(context=context) assert return_value == 0 - return_value = manage.delete_user(args=None) + return_value = manage.delete_user(args=None, context=context) assert return_value == 0 diff --git a/securedrop/tests/test_source.py b/securedrop/tests/test_source.py index f2179df960..ca3dec8e81 100644 --- a/securedrop/tests/test_source.py +++ b/securedrop/tests/test_source.py @@ -17,7 +17,6 @@ from mock import patch, ANY import pytest -import source from passphrases import PassphraseGenerator from source_app.session_manager import SessionManager from . import utils @@ -148,7 +147,7 @@ def test_generate(source_app): def test_create_duplicate_codename_logged_in_not_in_session(source_app): - with patch.object(source.app.logger, 'error') as logger: + with patch.object(source_app.logger, 'error') as logger: with source_app.test_client() as app: resp = app.get(url_for('main.generate')) assert resp.status_code == 200 @@ -329,21 +328,24 @@ def test_login_with_missing_reply_files(source_app): journalist, _ = utils.db_helper.init_journalist() replies = utils.db_helper.reply(journalist, source, 1) assert len(replies) > 0 + # Delete the reply file + reply_file_path = Path(source_app.storage.path(source.filesystem_id, replies[0].filename)) + reply_file_path.unlink() + assert not reply_file_path.exists() + with source_app.test_client() as app: - with patch("io.open") as ioMock: - ioMock.side_effect = FileNotFoundError - resp = app.get(url_for('main.login')) - assert resp.status_code == 200 - text = resp.data.decode('utf-8') - assert "Enter Codename" in text + resp = app.get(url_for('main.login')) + assert resp.status_code == 200 + text = resp.data.decode('utf-8') + assert "Enter Codename" in text - resp = app.post(url_for('main.login'), - data=dict(codename=codename), - follow_redirects=True) - assert resp.status_code == 200 - text = resp.data.decode('utf-8') - assert "Submit Files" in text - assert SessionManager.is_user_logged_in(db_session=db.session) + resp = app.post(url_for('main.login'), + data=dict(codename=codename), + follow_redirects=True) + assert resp.status_code == 200 + text = resp.data.decode('utf-8') + assert "Submit Files" in text + assert SessionManager.is_user_logged_in(db_session=db.session) def _dummy_submission(app): diff --git a/securedrop/tests/test_store.py b/securedrop/tests/test_store.py index 8993fc14b6..4d81b8973b 100644 --- a/securedrop/tests/test_store.py +++ b/securedrop/tests/test_store.py @@ -133,7 +133,7 @@ def test_verify_rejects_symlinks(journalist_app): def test_verify_store_dir_not_absolute(): with pytest.raises(store.PathException) as exc_info: - Storage('..', '/', '') + Storage('..', '/') msg = str(exc_info.value) assert re.compile('storage_path.*is not absolute').match(msg) @@ -141,7 +141,7 @@ def test_verify_store_dir_not_absolute(): def test_verify_store_temp_dir_not_absolute(): with pytest.raises(store.PathException) as exc_info: - Storage('/', '..', '') + Storage('/', '..') msg = str(exc_info.value) assert re.compile('temp_dir.*is not absolute').match(msg) diff --git a/securedrop/tests/utils/db_helper.py b/securedrop/tests/utils/db_helper.py index 5a891201ec..7c866682ea 100644 --- a/securedrop/tests/utils/db_helper.py +++ b/securedrop/tests/utils/db_helper.py @@ -13,10 +13,10 @@ from flask import current_app from db import db +from encryption import EncryptionManager from journalist_app.utils import mark_seen from models import Journalist, Reply, SeenReply, Submission from passphrases import PassphraseGenerator -from sdconfig import config from source_user import create_source_user @@ -74,11 +74,12 @@ def reply(journalist, source, num_replies): source.interaction_count += 1 fname = "{}-{}-reply.gpg".format(source.interaction_count, source.journalist_filename) - current_app.crypto_util.encrypt( - str(os.urandom(1)), - [current_app.crypto_util.get_fingerprint(source.filesystem_id), - config.JOURNALIST_KEY], - current_app.storage.path(source.filesystem_id, fname)) + + EncryptionManager.get_default().encrypt_journalist_reply( + for_source_with_filesystem_id=source.filesystem_id, + reply_in=str(os.urandom(1)), + encrypted_reply_path_out=current_app.storage.path(source.filesystem_id, fname), + ) reply = Reply(journalist, source, fname) replies.append(reply) @@ -133,7 +134,7 @@ def init_source(): source_passphrase=passphrase, source_app_storage=current_app.storage, ) - current_app.crypto_util.genkeypair(source_user) + EncryptionManager.get_default().generate_source_key_pair(source_user) return source_user.get_db_record(), passphrase