[wip] perf: speed up metadata sync time

dev-requirements.in

@@ -6,8 +6,8 @@ coverage==4.5.1
 flake8==3.6.0
 mccabe==0.6.1
 more-itertools==4.3.0
-mypy==0.701
-mypy-extensions==0.4.1
+mypy==0.761
+mypy-extensions==0.4.3
 pip-tools==4.2.0
 pluggy==0.13.0
 py==1.7.0
@@ -20,4 +20,4 @@ pytest-mock==1.10.0
 pytest-random-order==1.0.4
 pytest-xdist==1.30.0
 sip==4.19.8
-typed-ast==1.3.4
+typed-ast==1.4.1

dev-requirements.txt

@@ -86,21 +86,24 @@ more-itertools==4.3.0 \
     --hash=sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092 \
     --hash=sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e \
     --hash=sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d
-mypy-extensions==0.4.1 \
-    --hash=sha256:37e0e956f41369209a3d5f34580150bcacfabaa57b33a15c0b25f4b5725e0812 \
-    --hash=sha256:b16cabe759f55e3409a7d231ebd2841378fb0c27a5d1994719e340e4f429ac3e
-mypy==0.701 \
-    --hash=sha256:2afe51527b1f6cdc4a5f34fc90473109b22bf7f21086ba3e9451857cf11489e6 \
-    --hash=sha256:56a16df3e0abb145d8accd5dbb70eba6c4bd26e2f89042b491faa78c9635d1e2 \
-    --hash=sha256:5764f10d27b2e93c84f70af5778941b8f4aa1379b2430f85c827e0f5464e8714 \
-    --hash=sha256:5bbc86374f04a3aa817622f98e40375ccb28c4836f36b66706cf3c6ccce86eda \
-    --hash=sha256:6a9343089f6377e71e20ca734cd8e7ac25d36478a9df580efabfe9059819bf82 \
-    --hash=sha256:6c9851bc4a23dc1d854d3f5dfd5f20a016f8da86bcdbb42687879bb5f86434b0 \
-    --hash=sha256:b8e85956af3fcf043d6f87c91cbe8705073fc67029ba6e22d3468bfee42c4823 \
-    --hash=sha256:b9a0af8fae490306bc112229000aa0c2ccc837b49d29a5c42e088c132a2334dd \
-    --hash=sha256:bbf643528e2a55df2c1587008d6e3bda5c0445f1240dfa85129af22ae16d7a9a \
-    --hash=sha256:c46ab3438bd21511db0f2c612d89d8344154c0c9494afc7fbc932de514cf8d15 \
-    --hash=sha256:f7a83d6bd805855ef83ec605eb01ab4fa42bcef254b13631e451cbb44914a9b0
+mypy-extensions==0.4.3 \
+    --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \
+    --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8
+mypy==0.761 \
+    --hash=sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a \
+    --hash=sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7 \
+    --hash=sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2 \
+    --hash=sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474 \
+    --hash=sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0 \
+    --hash=sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217 \
+    --hash=sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749 \
+    --hash=sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6 \
+    --hash=sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf \
+    --hash=sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36 \
+    --hash=sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b \
+    --hash=sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72 \
+    --hash=sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1 \
+    --hash=sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1
 packaging==19.2 \
     --hash=sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47 \
     --hash=sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108 \
@@ -197,27 +200,33 @@ six==1.11.0 \
     --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb
 sqlalchemy==1.3.3 \
     --hash=sha256:91c54ca8345008fceaec987e10924bf07dcab36c442925357e5a467b36a38319
-typed-ast==1.3.4 \
-    --hash=sha256:04894d268ba6eab7e093d43107869ad49e7b5ef40d1a94243ea49b352061b200 \
-    --hash=sha256:16616ece19daddc586e499a3d2f560302c11f122b9c692bc216e821ae32aa0d0 \
-    --hash=sha256:252fdae740964b2d3cdfb3f84dcb4d6247a48a6abe2579e8029ab3be3cdc026c \
-    --hash=sha256:2af80a373af123d0b9f44941a46df67ef0ff7a60f95872412a145f4500a7fc99 \
-    --hash=sha256:2c88d0a913229a06282b285f42a31e063c3bf9071ff65c5ea4c12acb6977c6a7 \
-    --hash=sha256:2ea99c029ebd4b5a308d915cc7fb95b8e1201d60b065450d5d26deb65d3f2bc1 \
-    --hash=sha256:3d2e3ab175fc097d2a51c7a0d3fda442f35ebcc93bb1d7bd9b95ad893e44c04d \
-    --hash=sha256:4766dd695548a15ee766927bf883fb90c6ac8321be5a60c141f18628fb7f8da8 \
-    --hash=sha256:56b6978798502ef66625a2e0f80cf923da64e328da8bbe16c1ff928c70c873de \
-    --hash=sha256:5cddb6f8bce14325b2863f9d5ac5c51e07b71b462361fd815d1d7706d3a9d682 \
-    --hash=sha256:644ee788222d81555af543b70a1098f2025db38eaa99226f3a75a6854924d4db \
-    --hash=sha256:64cf762049fc4775efe6b27161467e76d0ba145862802a65eefc8879086fc6f8 \
-    --hash=sha256:68c362848d9fb71d3c3e5f43c09974a0ae319144634e7a47db62f0f2a54a7fa7 \
-    --hash=sha256:6c1f3c6f6635e611d58e467bf4371883568f0de9ccc4606f17048142dec14a1f \
-    --hash=sha256:b213d4a02eec4ddf622f4d2fbc539f062af3788d1f332f028a2e19c42da53f15 \
-    --hash=sha256:bb27d4e7805a7de0e35bd0cb1411bc85f807968b2b0539597a49a23b00a622ae \
-    --hash=sha256:c9d414512eaa417aadae7758bc118868cd2396b0e6138c1dd4fda96679c079d3 \
-    --hash=sha256:f0937165d1e25477b01081c4763d2d9cdc3b18af69cb259dd4f640c9b900fe5e \
-    --hash=sha256:fb96a6e2c11059ecf84e6741a319f93f683e440e341d4489c9b161eca251cf2a \
-    --hash=sha256:fc71d2d6ae56a091a8d94f33ec9d0f2001d1cb1db423d8b4355debfe9ce689b7
+typed-ast==1.4.1 \
+    --hash=sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355 \
+    --hash=sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919 \
+    --hash=sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa \
+    --hash=sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652 \
+    --hash=sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75 \
+    --hash=sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01 \
+    --hash=sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d \
+    --hash=sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1 \
+    --hash=sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907 \
+    --hash=sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c \
+    --hash=sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3 \
+    --hash=sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b \
+    --hash=sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614 \
+    --hash=sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb \
+    --hash=sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b \
+    --hash=sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41 \
+    --hash=sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6 \
+    --hash=sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34 \
+    --hash=sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe \
+    --hash=sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4 \
+    --hash=sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7
+typing-extensions==3.7.4.1 \
+    --hash=sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2 \
+    --hash=sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d \
+    --hash=sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575 \
+    # via mypy
 urllib3==1.24.3 \
     --hash=sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4 \
     --hash=sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb

securedrop_client/api_jobs/base.py

@@ -91,3 +91,20 @@ def call_api(self, api_client: API, session: Session) -> Any:
         failure.
         '''
         raise NotImplementedError
+
+
+class SingleObjectApiJob(ApiJob):
+    def __init__(self, uuid: str, remaining_attempts: int = DEFAULT_NUM_ATTEMPTS) -> None:
+        super().__init__(remaining_attempts)
+        '''
+        UUID of the item (source, reply, submission, etc.) that this item
+        corresponds to. We track this to prevent the addition of duplicate jobs.
+        '''
+        self.uuid = uuid
+
+    def __eq__(self, other: Any) -> bool:  # type: ignore[override]
+        # https://github.com/python/mypy/issues/2783
+        if self.uuid == getattr(other, 'uuid', None) and type(self) == type(other):
+            return True
+        else:
+            return False

securedrop_client/api_jobs/downloads.py

@@ -13,7 +13,7 @@
 from sdclientapi import Submission as SdkSubmission
 from sqlalchemy.orm.session import Session
 
-from securedrop_client.api_jobs.base import ApiJob
+from securedrop_client.api_jobs.base import SingleObjectApiJob
 from securedrop_client.crypto import GpgHelper, CryptoError
 from securedrop_client.db import File, Message, Reply
 from securedrop_client.storage import mark_as_decrypted, mark_as_downloaded, \
@@ -44,15 +44,15 @@ class DownloadDecryptionException(DownloadException):
     """
 
 
-class DownloadJob(ApiJob):
+class DownloadJob(SingleObjectApiJob):
     '''
     Download and decrypt a file that contains either a message, reply, or file submission.
     '''
 
     CHUNK_SIZE = 4096
 
-    def __init__(self, data_dir: str) -> None:
-        super().__init__()
+    def __init__(self, data_dir: str, uuid: str) -> None:
+        super().__init__(uuid)
         self.data_dir = data_dir
 
     def _get_realistic_timeout(self, size_in_bytes: int) -> int:
@@ -223,8 +223,7 @@ class ReplyDownloadJob(DownloadJob):
     '''
 
     def __init__(self, uuid: str, data_dir: str, gpg: GpgHelper) -> None:
-        super().__init__(data_dir)
-        self.uuid = uuid
+        super().__init__(data_dir, uuid)
         self.gpg = gpg
 
     def get_db_object(self, session: Session) -> Reply:
@@ -283,7 +282,7 @@ class MessageDownloadJob(DownloadJob):
     '''
 
     def __init__(self, uuid: str, data_dir: str, gpg: GpgHelper) -> None:
-        super().__init__(data_dir)
+        super().__init__(uuid, data_dir)
         self.uuid = uuid
         self.gpg = gpg
 
@@ -339,8 +338,7 @@ class FileDownloadJob(DownloadJob):
     '''
 
     def __init__(self, uuid: str, data_dir: str, gpg: GpgHelper) -> None:
-        super().__init__(data_dir)
-        self.uuid = uuid
+        super().__init__(uuid, data_dir)
         self.gpg = gpg
 
     def get_db_object(self, session: Session) -> File:

securedrop_client/api_jobs/sources.py

@@ -4,15 +4,14 @@
 from sdclientapi import API
 from sqlalchemy.orm.session import Session
 
-from securedrop_client.api_jobs.base import ApiJob
+from securedrop_client.api_jobs.base import SingleObjectApiJob
 
 logger = logging.getLogger(__name__)
 
 
-class DeleteSourceJob(ApiJob):
-    def __init__(self, source_uuid: str) -> None:
-        super().__init__()
-        self.source_uuid = source_uuid
+class DeleteSourceJob(SingleObjectApiJob):
+    def __init__(self, uuid: str) -> None:
+        super().__init__(uuid)
 
     def call_api(self, api_client: API, session: Session) -> str:
         '''
@@ -21,7 +20,7 @@ def call_api(self, api_client: API, session: Session) -> str:
         Delete a source on the server
         '''
         try:
-            source_sdk_object = sdclientapi.Source(uuid=self.source_uuid)
+            source_sdk_object = sdclientapi.Source(uuid=self.uuid)
 
             # TODO: After
             # https://github.com/freedomofpress/securedrop-client/issues/648
@@ -31,11 +30,11 @@ def call_api(self, api_client: API, session: Session) -> str:
             api_client.default_request_timeout = 5
             api_client.delete_source(source_sdk_object)
 
-            return self.source_uuid
+            return self.uuid
         except Exception as e:
             error_message = "Failed to delete source {uuid} due to {exception}".format(
-                uuid=self.source_uuid, exception=repr(e))
-            raise DeleteSourceJobException(error_message, self.source_uuid)
+                uuid=self.uuid, exception=repr(e))
+            raise DeleteSourceJobException(error_message, self.uuid)
 
 
 class DeleteSourceJobException(Exception):

securedrop_client/api_jobs/updatestar.py

@@ -6,14 +6,14 @@
 from sdclientapi import API
 from sqlalchemy.orm.session import Session
 
-from securedrop_client.api_jobs.base import ApiJob
+from securedrop_client.api_jobs.base import SingleObjectApiJob
 
 logger = logging.getLogger(__name__)
 
 
-class UpdateStarJob(ApiJob):
+class UpdateStarJob(SingleObjectApiJob):
     def __init__(self, source_uuid: str, star_status: bool) -> None:
-        super().__init__()
+        super().__init__(source_uuid)
         self.source_uuid = source_uuid
         self.star_status = star_status
 

securedrop_client/api_jobs/uploads.py

@@ -5,17 +5,17 @@
 from sqlalchemy.orm.session import Session
 from sqlalchemy.exc import SQLAlchemyError
 
-from securedrop_client.api_jobs.base import ApiJob
+from securedrop_client.api_jobs.base import SingleObjectApiJob
 from securedrop_client.crypto import GpgHelper
 from securedrop_client.db import DraftReply, Reply, ReplySendStatus, ReplySendStatusCodes, Source
 from securedrop_client.storage import update_draft_replies
 
 logger = logging.getLogger(__name__)
 
 
-class SendReplyJob(ApiJob):
+class SendReplyJob(SingleObjectApiJob):
     def __init__(self, source_uuid: str, reply_uuid: str, message: str, gpg: GpgHelper) -> None:
-        super().__init__()
+        super().__init__(reply_uuid)
         self.source_uuid = source_uuid
         self.reply_uuid = reply_uuid
         self.message = message

securedrop_client/queue.py

@@ -75,6 +75,12 @@ def add_job(self, job: ApiJob) -> None:
         '''
         Add the job with its priority to the queue after assigning it the next order_number.
         '''
+        in_progress_jobs = [in_progress_job for priority, in_progress_job in self.queue.queue]
+        if job in in_progress_jobs:
+            logger.debug('Duplicate job {}, skipping'.format(job))
+            return
+
+        logger.debug('Added {} to queue'.format(job.__class__.__name__))
         current_order_number = next(self.order_number)
         job.order_number = current_order_number
         priority = self.JOB_PRIORITIES[type(job)]
@@ -85,6 +91,7 @@ def re_add_job(self, job: ApiJob) -> None:
         Reset the job's remaining attempts and put it back into the queue in the order in which it
         was submitted by the user (do not assign it the next order_number).
         '''
+        logger.debug('Added {} to queue'.format(job.__class__.__name__))
         job.remaining_attempts = DEFAULT_NUM_ATTEMPTS
         priority = self.JOB_PRIORITIES[type(job)]
         self.queue.put_nowait((priority, job))
@@ -230,7 +237,5 @@ def enqueue(self, job: ApiJob) -> None:
 
         if isinstance(job, FileDownloadJob):
             self.download_file_queue.add_job(job)
-            logger.debug('Added {} to download queue'.format(job.__class__.__name__))
         else:
             self.main_queue.add_job(job)
-            logger.debug('Added {} to main queue'.format(job.__class__.__name__))

securedrop_client/sync.py

@@ -14,14 +14,14 @@
 
 class ApiSync(QObject):
     '''
-    ApiSync continuously syncs, waiting 15 seconds between task completion.
+    ApiSync continuously syncs, waiting 2 seconds between task completion.
     '''
 
     sync_started = pyqtSignal()
     sync_success = pyqtSignal()
     sync_failure = pyqtSignal(Exception)
 
-    TIME_BETWEEN_SYNCS_MS = 1000 * 15  # fifteen seconds between syncs
+    TIME_BETWEEN_SYNCS_MS = 1000 * 2  # two seconds between syncs
 
     def __init__(
         self, api_client: API, session_maker: scoped_session, gpg: GpgHelper, data_dir: str