Merge pull request #255 from freedomofpress/split-submission-table

Split submissions table into files/messages

README.md

@@ -100,7 +100,6 @@ rm -f svs.sqlite
 sqlite3 svs.sqlite .databases > /dev/null
 alembic upgrade head
 alembic revision --autogenerate -m "describe your revision here"
-make test-alembic
 ```
 
 ### Merging Migrations

alembic/versions/2f363b3d680e_init.py

@@ -23,17 +23,16 @@ def upgrade():
         sa.Column('uuid', sa.String(length=36), nullable=False),
         sa.Column('journalist_designation', sa.String(length=255), nullable=False),
         sa.Column('document_count', sa.Integer(), server_default='0', nullable=False),
-        sa.Column('is_flagged', sa.Boolean(name='is_flagged'), server_default='false',
-                  nullable=True),
+        sa.Column('is_flagged', sa.Boolean(name='is_flagged'), server_default='0', nullable=True),
         sa.Column('public_key', sa.Text(), nullable=True),
         sa.Column('fingerprint', sa.String(length=64), nullable=True),
         sa.Column('interaction_count', sa.Integer(), server_default='0', nullable=False),
-        sa.Column('is_starred', sa.Boolean(name='is_starred'), server_default='false',
-                  nullable=True),
+        sa.Column('is_starred', sa.Boolean(name='is_starred'), server_default='0', nullable=True),
         sa.Column('last_updated', sa.DateTime(), nullable=True),
         sa.PrimaryKeyConstraint('id', name=op.f('pk_sources')),
         sa.UniqueConstraint('uuid', name=op.f('uq_sources_uuid'))
     )
+
     op.create_table(
         'users',
         sa.Column('id', sa.Integer(), nullable=False),
@@ -42,6 +41,44 @@ def upgrade():
         sa.PrimaryKeyConstraint('id', name=op.f('pk_users')),
         sa.UniqueConstraint('uuid', name=op.f('uq_users_uuid'))
     )
+
+    op.create_table(
+        'files',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('uuid', sa.String(length=36), nullable=False),
+        sa.Column('filename', sa.String(length=255), nullable=False),
+        sa.Column('size', sa.Integer(), nullable=False),
+        sa.Column('download_url', sa.String(length=255), nullable=False),
+        sa.Column('is_downloaded', sa.Boolean(name='is_downloaded'), server_default='0',
+                  nullable=False),
+        sa.Column('is_read', sa.Boolean(name='is_read'), server_default='0', nullable=False),
+        sa.Column('source_id', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['source_id'], ['sources.id'],
+                                name=op.f('fk_files_source_id_sources')),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_files')),
+        sa.UniqueConstraint('uuid', name=op.f('uq_files_uuid')),
+    )
+
+    op.create_table(
+        'messages',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('uuid', sa.String(length=36), nullable=False),
+        sa.Column('filename', sa.String(length=255), nullable=False),
+        sa.Column('size', sa.Integer(), nullable=False),
+        sa.Column('download_url', sa.String(length=255), nullable=False),
+        sa.Column('is_downloaded', sa.Boolean(name='is_downloaded'), server_default='0',
+                  nullable=False),
+        sa.Column('is_read', sa.Boolean(name='is_read'), server_default='0', nullable=False),
+        sa.Column('content', sa.Text(), nullable=True),
+        sa.Column('source_id', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['source_id'], ['sources.id'],
+                                name=op.f('fk_messages_source_id_sources')),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_messages')),
+        sa.UniqueConstraint('uuid', name=op.f('uq_messages_uuid')),
+        sa.CheckConstraint('CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END',
+                           name='compare_download_vs_content'),
+    )
+
     op.create_table(
         'replies',
         sa.Column('id', sa.Integer(), nullable=False),
@@ -58,25 +95,11 @@ def upgrade():
         sa.PrimaryKeyConstraint('id', name=op.f('pk_replies')),
         sa.UniqueConstraint('uuid', name=op.f('uq_replies_uuid'))
     )
-    op.create_table(
-        'submissions',
-        sa.Column('id', sa.Integer(), nullable=False),
-        sa.Column('uuid', sa.String(length=36), nullable=False),
-        sa.Column('filename', sa.String(length=255), nullable=False),
-        sa.Column('size', sa.Integer(), nullable=False),
-        sa.Column('download_url', sa.String(length=255), nullable=False),
-        sa.Column('is_downloaded', sa.Boolean(name='is_downloaded'), nullable=True),
-        sa.Column('is_read', sa.Boolean(name='is_read'), nullable=True),
-        sa.Column('source_id', sa.Integer(), nullable=True),
-        sa.ForeignKeyConstraint(['source_id'], ['sources.id'],
-                                name=op.f('fk_submissions_source_id_sources')),
-        sa.PrimaryKeyConstraint('id', name=op.f('pk_submissions')),
-        sa.UniqueConstraint('uuid', name=op.f('uq_submissions_uuid'))
-    )
 
 
 def downgrade():
-    op.drop_table('submissions')
     op.drop_table('replies')
+    op.drop_table('messages')
+    op.drop_table('files')
     op.drop_table('users')
     op.drop_table('sources')

securedrop_client/db.py

@@ -1,7 +1,7 @@
 import os
 
 from sqlalchemy import Boolean, Column, create_engine, DateTime, ForeignKey, Integer, String, \
-    Text, MetaData
+    Text, MetaData, CheckConstraint
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import relationship, backref
 
@@ -32,13 +32,11 @@ class Source(Base):
     uuid = Column(String(36), unique=True, nullable=False)
     journalist_designation = Column(String(255), nullable=False)
     document_count = Column(Integer, server_default="0", nullable=False)
-    is_flagged = Column(Boolean(name='is_flagged'),
-                        server_default="false")
+    is_flagged = Column(Boolean(name='is_flagged'), server_default="0")
     public_key = Column(Text, nullable=True)
     fingerprint = Column(String(64))
     interaction_count = Column(Integer, server_default="0", nullable=False)
-    is_starred = Column(Boolean(name='is_starred'),
-                        server_default="false")
+    is_starred = Column(Boolean(name='is_starred'), server_default="0")
     last_updated = Column(DateTime)
 
     def __init__(self, uuid, journalist_designation, is_flagged, public_key,
@@ -60,15 +58,16 @@ def collection(self):
         """Return the list of submissions and replies for this source, sorted
         in ascending order by the filename/interaction count."""
         collection = []
-        collection.extend(self.submissions)
+        collection.extend(self.messages)
+        collection.extend(self.files)
         collection.extend(self.replies)
         collection.sort(key=lambda x: int(x.filename.split('-')[0]))
         return collection
 
 
-class Submission(Base):
+class Message(Base):
 
-    __tablename__ = 'submissions'
+    __tablename__ = 'messages'
 
     id = Column(Integer, primary_key=True)
     uuid = Column(String(36), unique=True, nullable=False)
@@ -77,30 +76,50 @@ class Submission(Base):
     download_url = Column(String(255), nullable=False)
 
     # This is whether the submission has been downloaded in the local database.
-    is_downloaded = Column(Boolean(name='is_downloaded'),
-                           default=False)
+    is_downloaded = Column(Boolean(name='is_downloaded'), nullable=False, server_default="0")
 
     # This reflects read status stored on the server.
-    is_read = Column(Boolean(name='is_read'),
-                     default=False)
+    is_read = Column(Boolean(name='is_read'), nullable=False, server_default="0")
+
+    content = Column(
+        Text,
+        # this check contraint ensures the state of the DB is what one would expect
+        CheckConstraint('CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END',
+                        name='compare_download_vs_content')
+    )
 
     source_id = Column(Integer, ForeignKey('sources.id'))
     source = relationship("Source",
-                          backref=backref("submissions", order_by=id,
+                          backref=backref("messages", order_by=id,
                                           cascade="delete"))
 
-    def __init__(self, source, uuid, size, filename, download_url):
-        # ORM event catching _should_ have already initialized `self.data`
+    def __repr__(self):
+        return '<Message {}>'.format(self.filename)
 
-        self.source_id = source.id
-        self.uuid = uuid
-        self.size = size
-        self.filename = filename
-        self.download_url = download_url
-        self.is_download = False
+
+class File(Base):
+
+    __tablename__ = 'files'
+
+    id = Column(Integer, primary_key=True)
+    uuid = Column(String(36), unique=True, nullable=False)
+    filename = Column(String(255), nullable=False)
+    size = Column(Integer, nullable=False)
+    download_url = Column(String(255), nullable=False)
+
+    # This is whether the submission has been downloaded in the local database.
+    is_downloaded = Column(Boolean(name='is_downloaded'), nullable=False, server_default="0")
+
+    # This reflects read status stored on the server.
+    is_read = Column(Boolean(name='is_read'), nullable=False, server_default="0")
+
+    source_id = Column(Integer, ForeignKey('sources.id'))
+    source = relationship("Source",
+                          backref=backref("files", order_by=id,
+                                          cascade="delete"))
 
     def __repr__(self):
-        return '<Submission {}>'.format(self.filename)
+        return '<File {}>'.format(self.filename)
 
 
 class Reply(Base):

securedrop_client/logic.py

@@ -553,16 +553,14 @@ def set_status(self, message, duration=5000):
         """
         self.gui.set_status(message, duration)
 
-    def on_file_open(self, submission_db_object):
+    def on_file_open(self, file_db_object):
         """
-        Open the already downloaded file associated with the message (which
-        is a Submission).
+        Open the already downloaded file associated with the message (which is a `File`).
         """
-
         # Once downloaded, submissions are stored in the data directory
         # with the same filename as the server, except with the .gz.gpg
         # stripped off.
-        server_filename = submission_db_object.filename
+        server_filename = file_db_object.filename
         fn_no_ext, _ = os.path.splitext(os.path.splitext(server_filename)[0])
         submission_filepath = os.path.join(self.data_dir, fn_no_ext)
 
@@ -588,7 +586,7 @@ def on_file_download(self, source_db_object, message):
             self.on_action_requiring_login()
             return
 
-        if isinstance(message, db.Submission):
+        if isinstance(message, db.File) or isinstance(message, db.Message):
             # Handle submissions.
             func = self.api.download_submission
             sdk_object = sdclientapi.Submission(uuid=message.uuid)

securedrop_client/message_sync.py

@@ -26,7 +26,7 @@
 from PyQt5.QtCore import QObject, pyqtSignal
 from securedrop_client import storage
 from securedrop_client.crypto import GpgHelper
-from securedrop_client.db import make_engine
+from securedrop_client.db import make_engine, Message
 from securedrop_client.storage import get_data
 from sqlalchemy.orm import sessionmaker
 
@@ -70,7 +70,8 @@ def __init__(self, api, home, is_qubes):
 
     def run(self, loop=True):
         while True:
-            submissions = storage.find_new_submissions(self.session)
+            submissions = storage.find_new_messages(self.session)
+            submissions.extend(storage.find_new_files(self.session))
 
             for db_submission in submissions:
                 try:
@@ -81,11 +82,16 @@ def run(self, loop=True):
                     # Need to set filename on non-Qubes platforms
                     sdk_submission.filename = db_submission.filename
 
+                    if isinstance(db_submission, Message):
+                        callback = storage.mark_message_as_downloaded
+                    else:
+                        callback = storage.mark_file_as_downloaded
+
                     if self.api:
                         self.fetch_the_thing(sdk_submission,
                                              db_submission,
                                              self.api.download_submission,
-                                             storage.mark_file_as_downloaded)
+                                             callback)
                         self.message_downloaded.emit(db_submission.uuid,
                                                      get_data(self.home, db_submission.filename))
                 except Exception: