From 51b2066a3f01963f2f90e3c4ad2339ee747cbf66 Mon Sep 17 00:00:00 2001 From: redshiftzero Date: Mon, 3 Jun 2019 17:41:52 -0700 Subject: [PATCH] app: pure refactor/reorganize to separate *Queue and APIJob objects We are going to be adding quite a few more API jobs (even without considering any additional functionality that might be added to the application). Let's separate the logic out a bit so that it's easier to move around the codebase. --- Makefile | 5 +- securedrop_client/api_jobs/__init__.py | 0 securedrop_client/api_jobs/base.py | 62 +++++ securedrop_client/api_jobs/downloads.py | 111 ++++++++ securedrop_client/queue.py | 163 +----------- tests/api_jobs/__init__.py | 0 tests/api_jobs/test_base.py | 97 +++++++ tests/api_jobs/test_downloads.py | 208 +++++++++++++++ tests/factory.py | 22 ++ tests/test_queue.py | 331 +----------------------- 10 files changed, 516 insertions(+), 483 deletions(-) create mode 100644 securedrop_client/api_jobs/__init__.py create mode 100644 securedrop_client/api_jobs/base.py create mode 100644 securedrop_client/api_jobs/downloads.py create mode 100644 tests/api_jobs/__init__.py create mode 100644 tests/api_jobs/test_base.py create mode 100644 tests/api_jobs/test_downloads.py diff --git a/Makefile b/Makefile index ba0681675..e44b8bafc 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,10 @@ mypy: ## Run static type checker securedrop_client/resources/__init__.py \ securedrop_client/storage.py \ securedrop_client/message_sync.py \ - securedrop_client/queue.py + securedrop_client/queue.py \ + securedrop_client/api_jobs/__init__.py \ + securedrop_client/api_jobs/base.py \ + securedrop_client/api_jobs/downloads.py .PHONY: clean clean: ## Clean the workspace of generated resources diff --git a/securedrop_client/api_jobs/__init__.py b/securedrop_client/api_jobs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/securedrop_client/api_jobs/base.py b/securedrop_client/api_jobs/base.py new file mode 100644 index 000000000..fbde085b6 --- /dev/null +++ b/securedrop_client/api_jobs/base.py @@ -0,0 +1,62 @@ +import logging + +from PyQt5.QtCore import QObject, pyqtSignal +from sdclientapi import API, RequestTimeoutError, AuthError +from sqlalchemy.orm.session import Session +from typing import Any, Optional + + +logger = logging.getLogger(__name__) + + +class ApiInaccessibleError(Exception): + + def __init__(self, message: Optional[str] = None) -> None: + if not message: + message = ('API is inaccessible either because there is no client or because the ' + 'client is not properly authenticated.') + super().__init__(message) + + +class ApiJob(QObject): + + ''' + Signal that is emitted after an job finishes successfully. + ''' + success_signal = pyqtSignal('PyQt_PyObject') + + ''' + Signal that is emitted if there is a failure during the job. + ''' + failure_signal = pyqtSignal(Exception) + + def __init__(self) -> None: + super().__init__(None) # `None` because the QOjbect has no parent + + def _do_call_api(self, api_client: API, session: Session) -> None: + if not api_client: + raise ApiInaccessibleError() + + try: + result = self.call_api(api_client, session) + except AuthError as e: + raise ApiInaccessibleError() from e + except RequestTimeoutError: + logger.debug('Job {} timed out'.format(self)) + raise + except Exception as e: + logger.error('Job {} raised an exception: {}: {}' + .format(self, type(e).__name__, e)) + self.failure_signal.emit(e) + else: + self.success_signal.emit(result) + + def call_api(self, api_client: API, session: Session) -> Any: + ''' + Method for making the actual API call and handling the result. + + This MUST resturn a value if the API call and other tasks were successful and MUST raise + an exception if and only iff the tasks failed. Presence of a raise exception indicates a + failure. + ''' + raise NotImplementedError diff --git a/securedrop_client/api_jobs/downloads.py b/securedrop_client/api_jobs/downloads.py new file mode 100644 index 000000000..9bc1229c9 --- /dev/null +++ b/securedrop_client/api_jobs/downloads.py @@ -0,0 +1,111 @@ +import binascii +import hashlib +import logging +import os +import sdclientapi +import shutil + +from sdclientapi import API +from sqlalchemy.orm.session import Session +from typing import Any, Union, Type, Tuple + +from securedrop_client import storage +from securedrop_client.api_jobs.base import ApiJob +from securedrop_client.crypto import GpgHelper, CryptoError +from securedrop_client.db import File, Message + + +logger = logging.getLogger(__name__) + + +class DownloadSubmissionJob(ApiJob): + + CHUNK_SIZE = 4096 + + def __init__( + self, + submission_type: Union[Type[File], Type[Message]], + submission_uuid: str, + data_dir: str, + gpg: GpgHelper, + ) -> None: + super().__init__() + self.data_dir = data_dir + self.submission_type = submission_type + self.submission_uuid = submission_uuid + self.gpg = gpg + + def call_api(self, api_client: API, session: Session) -> Any: + db_object = session.query(self.submission_type) \ + .filter_by(uuid=self.submission_uuid).one() + + etag, download_path = self._make_call(db_object, api_client) + + if not self._check_file_integrity(etag, download_path): + raise RuntimeError('Downloaded file had an invalid checksum.') + + self._decrypt_file(session, db_object, download_path) + + return db_object.uuid + + def _make_call(self, db_object: Union[File, Message], api_client: API) -> Tuple[str, str]: + sdk_obj = sdclientapi.Submission(uuid=db_object.uuid) + sdk_obj.filename = db_object.filename + sdk_obj.source_uuid = db_object.source.uuid + + return api_client.download_submission(sdk_obj) + + @classmethod + def _check_file_integrity(cls, etag: str, file_path: str) -> bool: + ''' + Checks if the file is valid. + :return: `True` if valid or unknown, `False` otherwise. + ''' + if not etag: + logger.debug('No ETag. Skipping integrity check for file at {}'.format(file_path)) + return True + + alg, checksum = etag.split(':') + + if alg == 'sha256': + hasher = hashlib.sha256() + else: + logger.debug('Unknown hash algorithm ({}). Skipping integrity check for file at {}' + .format(alg, file_path)) + return True + + with open(file_path, 'rb') as f: + while True: + read_bytes = f.read(cls.CHUNK_SIZE) + if not read_bytes: + break + hasher.update(read_bytes) + + calculated_checksum = binascii.hexlify(hasher.digest()).decode('utf-8') + return calculated_checksum == checksum + + def _decrypt_file( + self, + session: Session, + db_object: Union[File, Message], + file_path: str, + ) -> None: + file_uuid = db_object.uuid + server_filename = db_object.filename + + # The filename contains the location where the file has been stored. On non-Qubes OSes, this + # will be the data directory. On Qubes OS, this will a ~/QubesIncoming directory. In case we + # are on Qubes, we should move the file to the data directory and name it the same as the + # server (e.g. spotless-tater-msg.gpg). + filepath_in_datadir = os.path.join(self.data_dir, server_filename) + shutil.move(file_path, filepath_in_datadir) + storage.mark_file_as_downloaded(file_uuid, session) + + try: + self.gpg.decrypt_submission_or_reply(filepath_in_datadir, server_filename, is_doc=True) + except CryptoError as e: + logger.debug('Failed to decrypt file {}: {}'.format(server_filename, e)) + storage.set_object_decryption_status_with_content(db_object, session, False) + raise e + + storage.set_object_decryption_status_with_content(db_object, session, True) diff --git a/securedrop_client/queue.py b/securedrop_client/queue.py index c0a78c19d..4b885df48 100644 --- a/securedrop_client/queue.py +++ b/securedrop_client/queue.py @@ -1,172 +1,19 @@ -import binascii -import hashlib import logging -import os -import sdclientapi -import shutil -from PyQt5.QtCore import QObject, QThread, pyqtSlot, pyqtSignal +from PyQt5.QtCore import QObject, QThread, pyqtSlot from PyQt5.QtWidgets import QApplication from queue import Queue -from sdclientapi import API, RequestTimeoutError, AuthError +from sdclientapi import API, RequestTimeoutError from sqlalchemy.orm import scoped_session -from sqlalchemy.orm.session import Session -from typing import Any, Union, Optional, Type, Tuple +from typing import Optional # noqa: F401 -from securedrop_client import storage -from securedrop_client.crypto import GpgHelper, CryptoError -from securedrop_client.db import File, Message +from securedrop_client.api_jobs.base import ApiJob +from securedrop_client.api_jobs.downloads import DownloadSubmissionJob logger = logging.getLogger(__name__) -class ApiInaccessibleError(Exception): - - def __init__(self, message: Optional[str] = None) -> None: - if not message: - message = ('API is inaccessible either because there is no client or because the ' - 'client is not properly authenticated.') - super().__init__(message) - - -class ApiJob(QObject): - - ''' - Signal that is emitted after an job finishes successfully. - ''' - success_signal = pyqtSignal('PyQt_PyObject') - - ''' - Signal that is emitted if there is a failure during the job. - ''' - failure_signal = pyqtSignal(Exception) - - def __init__(self) -> None: - super().__init__(None) # `None` because the QOjbect has no parent - - def _do_call_api(self, api_client: API, session: Session) -> None: - if not api_client: - raise ApiInaccessibleError() - - try: - result = self.call_api(api_client, session) - except AuthError as e: - raise ApiInaccessibleError() from e - except RequestTimeoutError: - logger.debug('Job {} timed out'.format(self)) - raise - except Exception as e: - logger.error('Job {} raised an exception: {}: {}' - .format(self, type(e).__name__, e)) - self.failure_signal.emit(e) - else: - self.success_signal.emit(result) - - def call_api(self, api_client: API, session: Session) -> Any: - ''' - Method for making the actual API call and handling the result. - - This MUST resturn a value if the API call and other tasks were successful and MUST raise - an exception if and only iff the tasks failed. Presence of a raise exception indicates a - failure. - ''' - raise NotImplementedError - - -class DownloadSubmissionJob(ApiJob): - - CHUNK_SIZE = 4096 - - def __init__( - self, - submission_type: Union[Type[File], Type[Message]], - submission_uuid: str, - data_dir: str, - gpg: GpgHelper, - ) -> None: - super().__init__() - self.data_dir = data_dir - self.submission_type = submission_type - self.submission_uuid = submission_uuid - self.gpg = gpg - - def call_api(self, api_client: API, session: Session) -> Any: - db_object = session.query(self.submission_type) \ - .filter_by(uuid=self.submission_uuid).one() - - etag, download_path = self._make_call(db_object, api_client) - - if not self._check_file_integrity(etag, download_path): - raise RuntimeError('Downloaded file had an invalid checksum.') - - self._decrypt_file(session, db_object, download_path) - - return db_object.uuid - - def _make_call(self, db_object: Union[File, Message], api_client: API) -> Tuple[str, str]: - sdk_obj = sdclientapi.Submission(uuid=db_object.uuid) - sdk_obj.filename = db_object.filename - sdk_obj.source_uuid = db_object.source.uuid - - return api_client.download_submission(sdk_obj) - - @classmethod - def _check_file_integrity(cls, etag: str, file_path: str) -> bool: - ''' - Checks if the file is valid. - :return: `True` if valid or unknown, `False` otherwise. - ''' - if not etag: - logger.debug('No ETag. Skipping integrity check for file at {}'.format(file_path)) - return True - - alg, checksum = etag.split(':') - - if alg == 'sha256': - hasher = hashlib.sha256() - else: - logger.debug('Unknown hash algorithm ({}). Skipping integrity check for file at {}' - .format(alg, file_path)) - return True - - with open(file_path, 'rb') as f: - while True: - read_bytes = f.read(cls.CHUNK_SIZE) - if not read_bytes: - break - hasher.update(read_bytes) - - calculated_checksum = binascii.hexlify(hasher.digest()).decode('utf-8') - return calculated_checksum == checksum - - def _decrypt_file( - self, - session: Session, - db_object: Union[File, Message], - file_path: str, - ) -> None: - file_uuid = db_object.uuid - server_filename = db_object.filename - - # The filename contains the location where the file has been stored. On non-Qubes OSes, this - # will be the data directory. On Qubes OS, this will a ~/QubesIncoming directory. In case we - # are on Qubes, we should move the file to the data directory and name it the same as the - # server (e.g. spotless-tater-msg.gpg). - filepath_in_datadir = os.path.join(self.data_dir, server_filename) - shutil.move(file_path, filepath_in_datadir) - storage.mark_file_as_downloaded(file_uuid, session) - - try: - self.gpg.decrypt_submission_or_reply(filepath_in_datadir, server_filename, is_doc=True) - except CryptoError as e: - logger.debug('Failed to decrypt file {}: {}'.format(server_filename, e)) - storage.set_object_decryption_status_with_content(db_object, session, False) - raise e - - storage.set_object_decryption_status_with_content(db_object, session, True) - - class RunnableQueue(QObject): def __init__(self, api_client: API, session_maker: scoped_session) -> None: diff --git a/tests/api_jobs/__init__.py b/tests/api_jobs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/api_jobs/test_base.py b/tests/api_jobs/test_base.py new file mode 100644 index 000000000..78a02a98b --- /dev/null +++ b/tests/api_jobs/test_base.py @@ -0,0 +1,97 @@ +import pytest + +from sdclientapi import AuthError, RequestTimeoutError + +from securedrop_client.api_jobs.base import ApiInaccessibleError, ApiJob +from tests.factory import dummy_job_factory + + +def test_ApiInaccessibleError_init(): + # check default value + err = ApiInaccessibleError() + assert str(err).startswith('API is inaccessible') + assert isinstance(err, Exception) + + # check custom + msg = 'foo' + err = ApiInaccessibleError(msg) + assert str(err) == msg + + +def test_ApiJob_raises_NotImplemetedError(): + job = ApiJob() + + with pytest.raises(NotImplementedError): + job.call_api(None, None) + + +def test_ApiJob_no_api(mocker): + return_value = 'wat' + api_job_cls = dummy_job_factory(mocker, return_value) + api_job = api_job_cls() + + mock_session = mocker.MagicMock() + + with pytest.raises(ApiInaccessibleError): + api_job._do_call_api(None, mock_session) + + assert not api_job.success_signal.emit.called + assert not api_job.failure_signal.emit.called + + +def test_ApiJob_success(mocker): + return_value = 'wat' + api_job_cls = dummy_job_factory(mocker, return_value) + api_job = api_job_cls() + + mock_api_client = mocker.MagicMock() + mock_session = mocker.MagicMock() + + api_job._do_call_api(mock_api_client, mock_session) + + api_job.success_signal.emit.assert_called_once_with(return_value) + assert not api_job.failure_signal.emit.called + + +def test_ApiJob_auth_error(mocker): + return_value = AuthError('oh no') + api_job_cls = dummy_job_factory(mocker, return_value) + api_job = api_job_cls() + + mock_api_client = mocker.MagicMock() + mock_session = mocker.MagicMock() + + with pytest.raises(ApiInaccessibleError): + api_job._do_call_api(mock_api_client, mock_session) + + assert not api_job.success_signal.emit.called + assert not api_job.failure_signal.emit.called + + +def test_ApiJob_timeout_error(mocker): + return_value = RequestTimeoutError() + api_job_cls = dummy_job_factory(mocker, return_value) + api_job = api_job_cls() + + mock_api_client = mocker.MagicMock() + mock_session = mocker.MagicMock() + + with pytest.raises(RequestTimeoutError): + api_job._do_call_api(mock_api_client, mock_session) + + assert not api_job.success_signal.emit.called + assert not api_job.failure_signal.emit.called + + +def test_ApiJob_other_error(mocker): + return_value = Exception() + api_job_cls = dummy_job_factory(mocker, return_value) + api_job = api_job_cls() + + mock_api_client = mocker.MagicMock() + mock_session = mocker.MagicMock() + + api_job._do_call_api(mock_api_client, mock_session) + + assert not api_job.success_signal.emit.called + api_job.failure_signal.emit.assert_called_once_with(return_value) diff --git a/tests/api_jobs/test_downloads.py b/tests/api_jobs/test_downloads.py new file mode 100644 index 000000000..531d07127 --- /dev/null +++ b/tests/api_jobs/test_downloads.py @@ -0,0 +1,208 @@ +import os +import pytest +import sdclientapi +from typing import Tuple + +from securedrop_client import db +from securedrop_client.api_jobs.downloads import DownloadSubmissionJob +from securedrop_client.crypto import GpgHelper, CryptoError +from tests import factory + + +def test_DownloadSubmissionJob_happy_path_no_etag(mocker, homedir, session, session_maker): + source = factory.Source() + file_ = factory.File(source=source) + session.add(source) + session.add(file_) + session.commit() + + gpg = GpgHelper(homedir, session_maker, is_qubes=False) + mock_decrypt = mocker.patch.object(gpg, 'decrypt_submission_or_reply') + + def fake_download(sdk_obj: sdclientapi.Submission) -> Tuple[str, str]: + ''' + :return: (etag, path_to_dl) + ''' + full_path = os.path.join(homedir, 'somepath') + with open(full_path, 'wb') as f: + f.write(b'') + return ('', full_path) + + api_client = mocker.MagicMock() + api_client.download_submission = fake_download + + job = DownloadSubmissionJob( + db.File, + file_.uuid, + homedir, + gpg, + ) + + mock_logger = mocker.patch('securedrop_client.api_jobs.downloads.logger') + + job.call_api(api_client, session) + + log_msg = mock_logger.debug.call_args_list[0][0][0] + assert log_msg.startswith('No ETag. Skipping integrity check') + + # ensure mocks aren't stale + assert mock_decrypt.called + + +def test_DownloadSubmissionJob_happy_path_sha256_etag(mocker, homedir, session, session_maker): + source = factory.Source() + file_ = factory.File(source=source) + session.add(source) + session.add(file_) + session.commit() + + gpg = GpgHelper(homedir, session_maker, is_qubes=False) + mock_decrypt = mocker.patch.object(gpg, 'decrypt_submission_or_reply') + + def fake_download(sdk_obj: sdclientapi.Submission) -> Tuple[str, str]: + ''' + :return: (etag, path_to_dl) + ''' + full_path = os.path.join(homedir, 'somepath') + with open(full_path, 'wb') as f: + f.write(b'wat') + + # sha256 of b'wat' + return ('sha256:f00a787f7492a95e165b470702f4fe9373583fbdc025b2c8bdf0262cc48fcff4', + full_path) + + api_client = mocker.MagicMock() + api_client.download_submission = fake_download + + job = DownloadSubmissionJob( + db.File, + file_.uuid, + homedir, + gpg, + ) + + job.call_api(api_client, session) + + # ensure mocks aren't stale + assert mock_decrypt.called + + +def test_DownloadSubmissionJob_bad_sha256_etag(mocker, homedir, session, session_maker): + source = factory.Source() + file_ = factory.File(source=source) + session.add(source) + session.add(file_) + session.commit() + + gpg = GpgHelper(homedir, session_maker, is_qubes=False) + + def fake_download(sdk_obj: sdclientapi.Submission) -> Tuple[str, str]: + ''' + :return: (etag, path_to_dl) + ''' + full_path = os.path.join(homedir, 'somepath') + with open(full_path, 'wb') as f: + f.write(b'') + + return ('sha256:not-a-sha-sum', + full_path) + + api_client = mocker.MagicMock() + api_client.download_submission = fake_download + + job = DownloadSubmissionJob( + db.File, + file_.uuid, + homedir, + gpg, + ) + + # we currently don't handle errors in the checksum + with pytest.raises(RuntimeError): + job.call_api(api_client, session) + + +def test_DownloadSubmissionJob_happy_path_unknown_etag(mocker, homedir, session, session_maker): + source = factory.Source() + file_ = factory.File(source=source) + session.add(source) + session.add(file_) + session.commit() + + gpg = GpgHelper(homedir, session_maker, is_qubes=False) + + def fake_download(sdk_obj: sdclientapi.Submission) -> Tuple[str, str]: + ''' + :return: (etag, path_to_dl) + ''' + full_path = os.path.join(homedir, 'somepath') + with open(full_path, 'wb') as f: + f.write(b'') + return ('UNKNOWN:abc123', + full_path) + + api_client = mocker.MagicMock() + api_client.download_submission = fake_download + + job = DownloadSubmissionJob( + db.File, + file_.uuid, + homedir, + gpg, + ) + + mock_decrypt = mocker.patch('securedrop_client.crypto.GpgHelper.decrypt_submission_or_reply') + mock_logger = mocker.patch('securedrop_client.api_jobs.downloads.logger') + + job.call_api(api_client, session) + + log_msg = mock_logger.debug.call_args_list[0][0][0] + assert log_msg.startswith('Unknown hash algorithm') + + # ensure mocks aren't stale + assert mock_decrypt.called + + +def test_DownloadSubmissionJob_decryption_error(mocker, homedir, session, session_maker): + source = factory.Source() + file_ = factory.File(source=source) + session.add(source) + session.add(file_) + session.commit() + + gpg = GpgHelper(homedir, session_maker, is_qubes=False) + mock_decrypt = mocker.patch.object(gpg, 'decrypt_submission_or_reply', + side_effect=CryptoError) + + def fake_download(sdk_obj: sdclientapi.Submission) -> Tuple[str, str]: + ''' + :return: (etag, path_to_dl) + ''' + full_path = os.path.join(homedir, 'somepath') + with open(full_path, 'wb') as f: + f.write(b'wat') + + # sha256 of b'wat' + return ('sha256:f00a787f7492a95e165b470702f4fe9373583fbdc025b2c8bdf0262cc48fcff4', + full_path) + + api_client = mocker.MagicMock() + api_client.download_submission = fake_download + + job = DownloadSubmissionJob( + db.File, + file_.uuid, + homedir, + gpg, + ) + + mock_logger = mocker.patch('securedrop_client.api_jobs.downloads.logger') + + with pytest.raises(CryptoError): + job.call_api(api_client, session) + + log_msg = mock_logger.debug.call_args_list[0][0][0] + assert log_msg.startswith('Failed to decrypt file') + + # ensure mocks aren't stale + assert mock_decrypt.called diff --git a/tests/factory.py b/tests/factory.py index 7c3a19b8f..e75ad350d 100644 --- a/tests/factory.py +++ b/tests/factory.py @@ -3,6 +3,7 @@ """ from datetime import datetime from securedrop_client import db +from securedrop_client.api_jobs.base import ApiJob SOURCE_COUNT = 0 MESSAGE_COUNT = 0 @@ -93,3 +94,24 @@ def File(**attrs): defaults.update(attrs) return db.File(**defaults) + + +def dummy_job_factory(mocker, return_value): + ''' + Factory that creates dummy `ApiJob`s to DRY up test code. + ''' + class DummyApiJob(ApiJob): + success_signal = mocker.MagicMock() + failure_signal = mocker.MagicMock() + + def __init__(self, *nargs, **kwargs): + super().__init__(*nargs, **kwargs) + self.return_value = return_value + + def call_api(self, api_client, session): + if isinstance(self.return_value, Exception): + raise self.return_value + else: + return self.return_value + + return DummyApiJob diff --git a/tests/test_queue.py b/tests/test_queue.py index 41362a9a1..18e9e34d3 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -1,131 +1,13 @@ ''' Testing for the ApiJobQueue and related classes. ''' -import os -import pytest -import sdclientapi - -from . import factory from queue import Queue -from sdclientapi import AuthError, RequestTimeoutError -from typing import Tuple +from sdclientapi import RequestTimeoutError from securedrop_client import db -from securedrop_client.crypto import GpgHelper, CryptoError -from securedrop_client.queue import ApiInaccessibleError, ApiJob, RunnableQueue, \ - DownloadSubmissionJob, ApiJobQueue - - -def test_ApiInaccessibleError_init(): - # check default value - err = ApiInaccessibleError() - assert str(err).startswith('API is inaccessible') - assert isinstance(err, Exception) - - # check custom - msg = 'foo' - err = ApiInaccessibleError(msg) - assert str(err) == msg - - -def test_ApiJob_raises_NotImplemetedError(): - job = ApiJob() - - with pytest.raises(NotImplementedError): - job.call_api(None, None) - - -def dummy_job_factory(mocker, return_value): - ''' - Factory that creates dummy `ApiJob`s to DRY up test code. - ''' - class DummyApiJob(ApiJob): - success_signal = mocker.MagicMock() - failure_signal = mocker.MagicMock() - - def __init__(self, *nargs, **kwargs): - super().__init__(*nargs, **kwargs) - self.return_value = return_value - - def call_api(self, api_client, session): - if isinstance(self.return_value, Exception): - raise self.return_value - else: - return self.return_value - - return DummyApiJob - - -def test_ApiJob_no_api(mocker): - return_value = 'wat' - api_job_cls = dummy_job_factory(mocker, return_value) - api_job = api_job_cls() - - mock_session = mocker.MagicMock() - - with pytest.raises(ApiInaccessibleError): - api_job._do_call_api(None, mock_session) - - assert not api_job.success_signal.emit.called - assert not api_job.failure_signal.emit.called - - -def test_ApiJob_success(mocker): - return_value = 'wat' - api_job_cls = dummy_job_factory(mocker, return_value) - api_job = api_job_cls() - - mock_api_client = mocker.MagicMock() - mock_session = mocker.MagicMock() - - api_job._do_call_api(mock_api_client, mock_session) - - api_job.success_signal.emit.assert_called_once_with(return_value) - assert not api_job.failure_signal.emit.called - - -def test_ApiJob_auth_error(mocker): - return_value = AuthError('oh no') - api_job_cls = dummy_job_factory(mocker, return_value) - api_job = api_job_cls() - - mock_api_client = mocker.MagicMock() - mock_session = mocker.MagicMock() - - with pytest.raises(ApiInaccessibleError): - api_job._do_call_api(mock_api_client, mock_session) - - assert not api_job.success_signal.emit.called - assert not api_job.failure_signal.emit.called - - -def test_ApiJob_timeout_error(mocker): - return_value = RequestTimeoutError() - api_job_cls = dummy_job_factory(mocker, return_value) - api_job = api_job_cls() - - mock_api_client = mocker.MagicMock() - mock_session = mocker.MagicMock() - - with pytest.raises(RequestTimeoutError): - api_job._do_call_api(mock_api_client, mock_session) - - assert not api_job.success_signal.emit.called - assert not api_job.failure_signal.emit.called - - -def test_ApiJob_other_error(mocker): - return_value = Exception() - api_job_cls = dummy_job_factory(mocker, return_value) - api_job = api_job_cls() - - mock_api_client = mocker.MagicMock() - mock_session = mocker.MagicMock() - - api_job._do_call_api(mock_api_client, mock_session) - - assert not api_job.success_signal.emit.called - api_job.failure_signal.emit.assert_called_once_with(return_value) +from securedrop_client.api_jobs.downloads import DownloadSubmissionJob +from securedrop_client.queue import RunnableQueue, ApiJobQueue +from tests import factory def test_RunnableQueue_init(mocker): @@ -150,7 +32,7 @@ def test_RunnableQueue_happy_path(mocker): mock_session_maker = mocker.MagicMock(return_value=mock_session) return_value = 'foo' - dummy_job_cls = dummy_job_factory(mocker, return_value) + dummy_job_cls = factory.dummy_job_factory(mocker, return_value) queue = RunnableQueue(mock_api_client, mock_session_maker) queue.queue.put_nowait(dummy_job_cls()) @@ -175,7 +57,7 @@ def test_RunnableQueue_job_timeout(mocker): mock_session_maker = mocker.MagicMock(return_value=mock_session) return_value = RequestTimeoutError() - dummy_job_cls = dummy_job_factory(mocker, return_value) + dummy_job_cls = factory.dummy_job_factory(mocker, return_value) job1 = dummy_job_cls() job2 = dummy_job_cls() @@ -211,205 +93,6 @@ def test_RunnableQueue_job_timeout(mocker): assert mock_process_events.called -def test_DownloadSubmissionJob_happy_path_no_etag(mocker, homedir, session, session_maker): - source = factory.Source() - file_ = factory.File(source=source) - session.add(source) - session.add(file_) - session.commit() - - gpg = GpgHelper(homedir, session_maker, is_qubes=False) - mock_decrypt = mocker.patch.object(gpg, 'decrypt_submission_or_reply') - - def fake_download(sdk_obj: sdclientapi.Submission) -> Tuple[str, str]: - ''' - :return: (etag, path_to_dl) - ''' - full_path = os.path.join(homedir, 'somepath') - with open(full_path, 'wb') as f: - f.write(b'') - return ('', full_path) - - api_client = mocker.MagicMock() - api_client.download_submission = fake_download - - job = DownloadSubmissionJob( - db.File, - file_.uuid, - homedir, - gpg, - ) - - mock_logger = mocker.patch('securedrop_client.queue.logger') - - job.call_api(api_client, session) - - log_msg = mock_logger.debug.call_args_list[0][0][0] - assert log_msg.startswith('No ETag. Skipping integrity check') - - # ensure mocks aren't stale - assert mock_decrypt.called - - -def test_DownloadSubmissionJob_happy_path_sha256_etag(mocker, homedir, session, session_maker): - source = factory.Source() - file_ = factory.File(source=source) - session.add(source) - session.add(file_) - session.commit() - - gpg = GpgHelper(homedir, session_maker, is_qubes=False) - mock_decrypt = mocker.patch.object(gpg, 'decrypt_submission_or_reply') - - def fake_download(sdk_obj: sdclientapi.Submission) -> Tuple[str, str]: - ''' - :return: (etag, path_to_dl) - ''' - full_path = os.path.join(homedir, 'somepath') - with open(full_path, 'wb') as f: - f.write(b'wat') - - # sha256 of b'wat' - return ('sha256:f00a787f7492a95e165b470702f4fe9373583fbdc025b2c8bdf0262cc48fcff4', - full_path) - - api_client = mocker.MagicMock() - api_client.download_submission = fake_download - - job = DownloadSubmissionJob( - db.File, - file_.uuid, - homedir, - gpg, - ) - - job.call_api(api_client, session) - - # ensure mocks aren't stale - assert mock_decrypt.called - - -def test_DownloadSubmissionJob_bad_sha256_etag(mocker, homedir, session, session_maker): - source = factory.Source() - file_ = factory.File(source=source) - session.add(source) - session.add(file_) - session.commit() - - gpg = GpgHelper(homedir, session_maker, is_qubes=False) - - def fake_download(sdk_obj: sdclientapi.Submission) -> Tuple[str, str]: - ''' - :return: (etag, path_to_dl) - ''' - full_path = os.path.join(homedir, 'somepath') - with open(full_path, 'wb') as f: - f.write(b'') - - return ('sha256:not-a-sha-sum', - full_path) - - api_client = mocker.MagicMock() - api_client.download_submission = fake_download - - job = DownloadSubmissionJob( - db.File, - file_.uuid, - homedir, - gpg, - ) - - # we currently don't handle errors in the checksum - with pytest.raises(RuntimeError): - job.call_api(api_client, session) - - -def test_DownloadSubmissionJob_happy_path_unknown_etag(mocker, homedir, session, session_maker): - source = factory.Source() - file_ = factory.File(source=source) - session.add(source) - session.add(file_) - session.commit() - - gpg = GpgHelper(homedir, session_maker, is_qubes=False) - - def fake_download(sdk_obj: sdclientapi.Submission) -> Tuple[str, str]: - ''' - :return: (etag, path_to_dl) - ''' - full_path = os.path.join(homedir, 'somepath') - with open(full_path, 'wb') as f: - f.write(b'') - return ('UNKNOWN:abc123', - full_path) - - api_client = mocker.MagicMock() - api_client.download_submission = fake_download - - job = DownloadSubmissionJob( - db.File, - file_.uuid, - homedir, - gpg, - ) - - mock_decrypt = mocker.patch('securedrop_client.crypto.GpgHelper.decrypt_submission_or_reply') - mock_logger = mocker.patch('securedrop_client.queue.logger') - - job.call_api(api_client, session) - - log_msg = mock_logger.debug.call_args_list[0][0][0] - assert log_msg.startswith('Unknown hash algorithm') - - # ensure mocks aren't stale - assert mock_decrypt.called - - -def test_DownloadSubmissionJob_decryption_error(mocker, homedir, session, session_maker): - source = factory.Source() - file_ = factory.File(source=source) - session.add(source) - session.add(file_) - session.commit() - - gpg = GpgHelper(homedir, session_maker, is_qubes=False) - mock_decrypt = mocker.patch.object(gpg, 'decrypt_submission_or_reply', - side_effect=CryptoError) - - def fake_download(sdk_obj: sdclientapi.Submission) -> Tuple[str, str]: - ''' - :return: (etag, path_to_dl) - ''' - full_path = os.path.join(homedir, 'somepath') - with open(full_path, 'wb') as f: - f.write(b'wat') - - # sha256 of b'wat' - return ('sha256:f00a787f7492a95e165b470702f4fe9373583fbdc025b2c8bdf0262cc48fcff4', - full_path) - - api_client = mocker.MagicMock() - api_client.download_submission = fake_download - - job = DownloadSubmissionJob( - db.File, - file_.uuid, - homedir, - gpg, - ) - - mock_logger = mocker.patch('securedrop_client.queue.logger') - - with pytest.raises(CryptoError): - job.call_api(api_client, session) - - log_msg = mock_logger.debug.call_args_list[0][0][0] - assert log_msg.startswith('Failed to decrypt file') - - # ensure mocks aren't stale - assert mock_decrypt.called - - def test_ApiJobQueue_enqueue(mocker): mock_client = mocker.MagicMock() mock_session_maker = mocker.MagicMock() @@ -428,7 +111,7 @@ def test_ApiJobQueue_enqueue(mocker): mock_download_queue.reset_mock() mock_main_queue.reset_mock() - dummy_job = dummy_job_factory(mocker, 'mock')() + dummy_job = factory.dummy_job_factory(mocker, 'mock')() job_queue.enqueue(dummy_job) mock_main_queue.queue.put_nowait.assert_called_once_with(dummy_job)