Merge pull request #277 from freedomofpress/store-messages-in-db

Store decrypted messages / replies in the DB

securedrop_client/db.py

@@ -39,17 +39,6 @@ class Source(Base):
     is_starred = Column(Boolean(name='is_starred'), server_default=text("0"))
     last_updated = Column(DateTime)
 
-    def __init__(self, uuid, journalist_designation, is_flagged, public_key,
-                 interaction_count, is_starred, last_updated, document_count):
-        self.uuid = uuid
-        self.journalist_designation = journalist_designation
-        self.is_flagged = is_flagged
-        self.public_key = public_key
-        self.document_count = document_count
-        self.interaction_count = interaction_count
-        self.is_starred = is_starred
-        self.last_updated = last_updated
-
     def __repr__(self):
         return '<Source {}>'.format(self.journalist_designation)
 

securedrop_client/gui/main.py

@@ -184,7 +184,6 @@ def show_conversation_for(self, source: Source, is_authenticated: bool):
         Show conversation of messages and replies between a source and
         journalists.
         """
-
         conversation_container = self.conversations.get(source.uuid, None)
 
         if conversation_container is None:

securedrop_client/gui/widgets.py

@@ -27,10 +27,9 @@
 from typing import List
 from uuid import uuid4
 
-from securedrop_client.db import Source, Message, File
+from securedrop_client.db import Source, Message, File, Reply
 from securedrop_client.logic import Client
 from securedrop_client.resources import load_svg, load_image
-from securedrop_client.storage import get_data
 from securedrop_client.utils import humanize_filesize
 
 logger = logging.getLogger(__name__)
@@ -727,25 +726,12 @@ def update_conversation(self, collection: list) -> None:
         # add new items
         for conversation_item in collection:
             if conversation_item.filename.endswith('msg.gpg'):
-                self.add_item_content_or(self.add_message,
-                                         conversation_item,
-                                         "<Message not yet downloaded>")
+                self.add_message(conversation_item)
             elif conversation_item.filename.endswith('reply.gpg'):
-                self.add_item_content_or(self.add_reply,
-                                         conversation_item,
-                                         "<Reply not yet downloaded>")
+                self.add_reply(conversation_item)
             else:
                 self.add_file(self.source, conversation_item)
 
-    def add_item_content_or(self, adder, item, default):
-        """
-        Private helper function to add correct message to conversation widgets
-        """
-        if item.is_downloaded is False:
-            adder(item.uuid, default)
-        else:
-            adder(item.uuid, get_data(self.sdc_home, item.filename))
-
     def add_file(self, source_db_object, submission_db_object):
         """
         Add a file from the source.
@@ -765,21 +751,33 @@ def update_conversation_position(self, min_val, max_val):
         if current_val + viewport_height > max_val:
             self.scroll.verticalScrollBar().setValue(max_val)
 
-    def add_message(self, message_id: str, message: str) -> None:
+    def add_message(self, message: Message) -> None:
         """
         Add a message from the source.
         """
+        self.controller.session.refresh(message)
+        if message.content is not None:
+            content = message.content
+        else:
+            content = '<Message not yet available>'
+
         self.conversation_layout.addWidget(
-            MessageWidget(message_id, message, self.controller.message_sync.message_downloaded))
+            MessageWidget(message.uuid, content, self.controller.message_sync.message_ready))
 
-    def add_reply(self, message_id: str, reply: str, files=None) -> None:
+    def add_reply(self, reply: Reply) -> None:
         """
         Add a reply from a journalist.
         """
+        self.controller.session.refresh(reply)
+        if reply.content is not None:
+            content = reply.content
+        else:
+            content = '<Reply not yet available>'
+
         self.conversation_layout.addWidget(
-            ReplyWidget(message_id,
-                        reply,
-                        self.controller.reply_sync.reply_downloaded,
+            ReplyWidget(reply.uuid,
+                        content,
+                        self.controller.reply_sync.reply_ready,
                         self.controller.reply_succeeded,
                         self.controller.reply_failed,
                         ))

securedrop_client/logic.py

@@ -626,10 +626,12 @@ def on_file_downloaded(self, result, current_object):
                 # Attempt to decrypt the file.
                 self.gpg.decrypt_submission_or_reply(
                     filepath_in_datadir, server_filename, is_doc=True)
-                storage.set_object_decryption_status(current_object, self.session, True)
+                storage.set_object_decryption_status_with_content(
+                    current_object, self.session, True)
             except CryptoError as e:
                 logger.debug('Failed to decrypt file {}: {}'.format(server_filename, e))
-                storage.set_object_decryption_status(current_object, self.session, False)
+                storage.set_object_decryption_status_with_content(
+                    current_object, self.session, False)
                 self.set_status("Failed to decrypt file, "
                                 "please try again or talk to your administrator.")
                 # TODO: We should save the downloaded content, and just

securedrop_client/message_sync.py

@@ -27,8 +27,8 @@
 from securedrop_client import storage
 from securedrop_client.crypto import GpgHelper, CryptoError
 from securedrop_client.db import make_engine
-from securedrop_client.storage import get_data
 from sqlalchemy.orm import sessionmaker
+from tempfile import NamedTemporaryFile
 
 
 logger = logging.getLogger(__name__)
@@ -52,13 +52,16 @@ def fetch_the_thing(self, item, msg, download_fn, update_fn):
         update_fn(msg.uuid, self.session)
         logger.info("Stored message or reply at {}".format(msg.filename))
 
-        try:
-            self.gpg.decrypt_submission_or_reply(filepath, msg.filename, False)
-            storage.set_object_decryption_status(msg, self.session, True)
-            logger.info("Message or reply decrypted: {}".format(msg.filename))
-        except CryptoError:
-            storage.set_object_decryption_status(msg, self.session, False)
-            logger.info("Message or reply failed to decrypt: {}".format(msg.filename))
+        with NamedTemporaryFile('w+') as plaintext_file:
+            try:
+                self.gpg.decrypt_submission_or_reply(filepath, plaintext_file.name, False)
+                plaintext_file.seek(0)
+                content = plaintext_file.read()
+                storage.set_object_decryption_status_with_content(msg, self.session, True, content)
+                logger.info("Message or reply decrypted: {}".format(msg.filename))
+            except CryptoError:
+                storage.set_object_decryption_status_with_content(msg, self.session, False)
+                logger.info("Message or reply failed to decrypt: {}".format(msg.filename))
 
 
 class MessageSync(APISyncObject):
@@ -67,10 +70,10 @@ class MessageSync(APISyncObject):
     """
 
     """
-    Signal emitted notifying that a message has been downloaded. The signal is a tuple of
+    Signal emitted notifying that a message is ready to be displayed. The signal is a tuple of
     (str, str) containing the message's UUID and the content of the message.
     """
-    message_downloaded = pyqtSignal([str, str])
+    message_ready = pyqtSignal([str, str])
 
     def __init__(self, api, home, is_qubes):
         super().__init__(api, home, is_qubes)
@@ -93,8 +96,13 @@ def run(self, loop=True):
                                              db_submission,
                                              self.api.download_submission,
                                              storage.mark_message_as_downloaded)
-                        self.message_downloaded.emit(db_submission.uuid,
-                                                     get_data(self.home, db_submission.filename))
+
+                        if db_submission.content is not None:
+                            content = db_submission.content
+                        else:
+                            content = '<Message not yet available>'
+
+                        self.message_ready.emit(db_submission.uuid, content)
                 except Exception:
                     tb = traceback.format_exc()
                     logger.critical("Exception while downloading submission!\n{}".format(tb))
@@ -113,10 +121,10 @@ class ReplySync(APISyncObject):
     """
 
     """
-    Signal emitted notifying that a reply has been downloaded. The signal is a tuple of
-    (str, str) containing the message's UUID and the content of the reply.
+    Signal emitted notifying that a reply is ready to be displayed. The signal is a tuple of
+    (str, str) containing the reply's UUID and the content of the reply.
     """
-    reply_downloaded = pyqtSignal([str, str])
+    reply_ready = pyqtSignal([str, str])
 
     def __init__(self, api, home, is_qubes):
         super().__init__(api, home, is_qubes)
@@ -142,8 +150,13 @@ def run(self, loop=True):
                                              db_reply,
                                              self.api.download_reply,
                                              storage.mark_reply_as_downloaded)
-                        self.reply_downloaded.emit(db_reply.uuid,
-                                                   get_data(self.home, db_reply.filename))
+
+                        if db_reply.content is not None:
+                            content = db_reply.content
+                        else:
+                            content = '<Reply not yet available>'
+
+                        self.reply_ready.emit(db_reply.uuid, content)
                 except Exception:
                     tb = traceback.format_exc()
                     logger.critical("Exception while downloading reply!\n{}".format(tb))

securedrop_client/storage.py

@@ -325,12 +325,14 @@ def mark_message_as_downloaded(uuid, session):
     session.commit()
 
 
-def set_object_decryption_status(obj, session, is_successful: bool):
+def set_object_decryption_status_with_content(obj, session, is_successful: bool, content=None):
     """Mark object as decrypted or not in the database."""
 
     model = type(obj)
     db_object = session.query(model).filter_by(uuid=obj.uuid).one_or_none()
     db_object.is_decrypted = is_successful
+    if content is not None:
+        db_object.content = content
     session.add(db_object)
     session.commit()
 
@@ -371,15 +373,3 @@ def rename_file(data_dir: str, filename: str, new_filename: str):
                   os.path.join(data_dir, new_filename))
     except OSError as e:
         logger.debug('File could not be renamed: {}'.format(e))
-
-
-def get_data(sdc_home: str, filename: str) -> str:
-    filename, _ = os.path.splitext(filename)
-    full_path = os.path.join(sdc_home, 'data', filename)
-    try:
-        with open(full_path) as f:
-            msg = f.read()
-    except FileNotFoundError:
-        logger.debug('File not found: {}'.format(full_path))
-        msg = '<Not Found>'
-    return msg

tests/gui/test_main.py

@@ -2,12 +2,11 @@
 Check the core Window UI class works as expected.
 """
 from PyQt5.QtWidgets import QApplication, QHBoxLayout
+from securedrop_client.db import Message
 from securedrop_client.gui.main import Window
 from securedrop_client.resources import load_icon
-from securedrop_client.db import Message
 from uuid import uuid4
 
-
 app = QApplication([])
 
 
@@ -261,7 +260,7 @@ def test_conversation_pending_message(mocker):
     w.show_conversation_for(mock_source, True)
 
     assert mocked_add_message.call_count == 1
-    assert mocked_add_message.call_args == mocker.call(msg_uuid, "<Message not yet downloaded>")
+    assert mocked_add_message.call_args == mocker.call(message)
 
 
 def test_set_status(mocker):