From c75bf83ceb3d43b962e05a1e10cfcdfa36cc2da5 Mon Sep 17 00:00:00 2001 From: Cory Francis Myers Date: Thu, 9 Jun 2022 16:28:51 -0700 Subject: [PATCH 01/10] refactor!: squash Alembic migrations as of 9ba8d7524871 BREAKING CHANGE: The "securedrop-client" entry-point will expect a nonexistent or empty database and migrate it to new revision d7c8af95bc8e. It will error out if a database already exists at another, prior revision. --- alembic/versions/2f363b3d680e_init.py | 161 ---------- ...fbfb_add_first_name_last_name_fullname_.py | 27 -- .../7f682532afa2_add_download_error.py | 302 ------------------ .../versions/86b01b6290da_add_reply_draft.py | 68 ---- .../9ba8d7524871_add_deletedsource_table.py | 32 -- ...9_fix_journalist_association_in_replies.py | 85 ----- alembic/versions/bafdcae12f97_.py | 31 -- .../versions/bd57477f19a2_add_seen_tables.py | 78 ----- alembic/versions/d7c8af95bc8e_initial.py | 256 +++++++++++++++ ...1387cfd0b_add_deletedconversation_table.py | 34 -- ...b657f2ee8a7_drop_file_original_filename.py | 72 ----- ..._remove_decryption_vs_content_contraint.py | 218 ------------- 12 files changed, 256 insertions(+), 1108 deletions(-) delete mode 100644 alembic/versions/2f363b3d680e_init.py delete mode 100644 alembic/versions/36a79ffcfbfb_add_first_name_last_name_fullname_.py delete mode 100644 alembic/versions/7f682532afa2_add_download_error.py delete mode 100644 alembic/versions/86b01b6290da_add_reply_draft.py delete mode 100644 alembic/versions/9ba8d7524871_add_deletedsource_table.py delete mode 100644 alembic/versions/a4bf1f58ce69_fix_journalist_association_in_replies.py delete mode 100644 alembic/versions/bafdcae12f97_.py delete mode 100644 alembic/versions/bd57477f19a2_add_seen_tables.py create mode 100644 alembic/versions/d7c8af95bc8e_initial.py delete mode 100644 alembic/versions/eff1387cfd0b_add_deletedconversation_table.py delete mode 100644 alembic/versions/fb657f2ee8a7_drop_file_original_filename.py delete mode 100644 alembic/versions/fecf1191b6f0_remove_decryption_vs_content_contraint.py diff --git a/alembic/versions/2f363b3d680e_init.py b/alembic/versions/2f363b3d680e_init.py deleted file mode 100644 index 8e1b2cede..000000000 --- a/alembic/versions/2f363b3d680e_init.py +++ /dev/null @@ -1,161 +0,0 @@ -"""init - -Revision ID: 2f363b3d680e -Revises: -Create Date: 2019-02-08 12:07:47.062397 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "2f363b3d680e" -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - "sources", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("journalist_designation", sa.String(length=255), nullable=False), - sa.Column("document_count", sa.Integer(), server_default=sa.text("0"), nullable=False), - sa.Column( - "is_flagged", sa.Boolean(name="is_flagged"), server_default=sa.text("0"), nullable=True - ), - sa.Column("public_key", sa.Text(), nullable=True), - sa.Column("fingerprint", sa.String(length=64), nullable=True), - sa.Column("interaction_count", sa.Integer(), server_default=sa.text("0"), nullable=False), - sa.Column( - "is_starred", sa.Boolean(name="is_starred"), server_default=sa.text("0"), nullable=True - ), - sa.Column("last_updated", sa.DateTime(), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_sources")), - sa.UniqueConstraint("uuid", name=op.f("uq_sources_uuid")), - ) - - op.create_table( - "users", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("username", sa.String(length=255), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_users")), - sa.UniqueConstraint("uuid", name=op.f("uq_users_uuid")), - ) - - op.create_table( - "files", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=False), - sa.Column("download_url", sa.String(length=255), nullable=False), - sa.Column( - "is_downloaded", - sa.Boolean(name="is_downloaded"), - server_default=sa.text("0"), - nullable=False, - ), - sa.Column( - "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False - ), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_files_source_id_sources") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_files")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_files_uuid")), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="files_compare_is_downloaded_vs_is_decrypted", - ), - ) - - op.create_table( - "messages", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=False), - sa.Column("download_url", sa.String(length=255), nullable=False), - sa.Column( - "is_downloaded", - sa.Boolean(name="is_downloaded"), - server_default=sa.text("0"), - nullable=False, - ), - sa.Column( - "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False - ), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_messages_source_id_sources") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_messages_uuid")), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name=op.f("ck_message_compare_download_vs_content"), - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="messages_compare_is_downloaded_vs_is_decrypted", - ), - sa.CheckConstraint( - "CASE WHEN is_decrypted = 0 THEN content IS NULL ELSE 1 END", - name="messages_compare_is_decrypted_vs_content", - ), - ) - - op.create_table( - "replies", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=True), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_replies_journalist_id_users") - ), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_replies_source_id_sources") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_replies")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_replies_uuid")), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name="replies_compare_download_vs_content", - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="replies_compare_is_downloaded_vs_is_decrypted", - ), - sa.CheckConstraint( - "CASE WHEN is_decrypted = 0 THEN content IS NULL ELSE 1 END", - name="replies_compare_is_decrypted_vs_content", - ), - ) - - -def downgrade(): - op.drop_table("replies") - op.drop_table("messages") - op.drop_table("files") - op.drop_table("users") - op.drop_table("sources") diff --git a/alembic/versions/36a79ffcfbfb_add_first_name_last_name_fullname_.py b/alembic/versions/36a79ffcfbfb_add_first_name_last_name_fullname_.py deleted file mode 100644 index 302153fba..000000000 --- a/alembic/versions/36a79ffcfbfb_add_first_name_last_name_fullname_.py +++ /dev/null @@ -1,27 +0,0 @@ -"""add first_name, last_name, fullname, initials - -Revision ID: 36a79ffcfbfb -Revises: fecf1191b6f0 -Create Date: 2019-06-28 15:21:50.893256 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "36a79ffcfbfb" -down_revision = "bafdcae12f97" -branch_labels = None -depends_on = None - - -def upgrade(): - op.add_column("users", sa.Column("firstname", sa.String(length=64), nullable=True)) - op.add_column("users", sa.Column("lastname", sa.String(length=64), nullable=True)) - - -def downgrade(): - with op.batch_alter_table("users", schema=None) as batch_op: - batch_op.drop_column("lastname") - batch_op.drop_column("firstname") diff --git a/alembic/versions/7f682532afa2_add_download_error.py b/alembic/versions/7f682532afa2_add_download_error.py deleted file mode 100644 index b0982f846..000000000 --- a/alembic/versions/7f682532afa2_add_download_error.py +++ /dev/null @@ -1,302 +0,0 @@ -"""add download error - -Revision ID: 7f682532afa2 -Revises: fb657f2ee8a7 -Create Date: 2020-04-15 13:44:21.434312 - -""" -import sqlalchemy as sa - -from alembic import op -from securedrop_client import db - -# revision identifiers, used by Alembic. -revision = "7f682532afa2" -down_revision = "fb657f2ee8a7" -branch_labels = None -depends_on = None - - -CREATE_TABLE_FILES_NEW = """ - CREATE TABLE files ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER NOT NULL, - download_url VARCHAR(255) NOT NULL, - is_downloaded BOOLEAN DEFAULT 0 NOT NULL, - is_decrypted BOOLEAN CONSTRAINT files_compare_is_downloaded_vs_is_decrypted CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - download_error_id INTEGER, - is_read BOOLEAN DEFAULT 0 NOT NULL, - source_id INTEGER NOT NULL, - last_updated DATETIME NOT NULL, - CONSTRAINT pk_files PRIMARY KEY (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_files_uuid UNIQUE (uuid), - CONSTRAINT ck_files_is_downloaded CHECK (is_downloaded IN (0, 1)), - CONSTRAINT ck_files_is_decrypted CHECK (is_decrypted IN (0, 1)), - CONSTRAINT fk_files_download_error_id_downloaderrors FOREIGN KEY(download_error_id) REFERENCES downloaderrors (id), - CONSTRAINT ck_files_is_read CHECK (is_read IN (0, 1)), - CONSTRAINT fk_files_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id) -); -""" - -CREATE_TABLE_FILES_OLD = """ - CREATE TABLE files ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER NOT NULL, - download_url VARCHAR(255) NOT NULL, - is_downloaded BOOLEAN DEFAULT 0 NOT NULL, - is_read BOOLEAN DEFAULT 0 NOT NULL, - is_decrypted BOOLEAN, - source_id INTEGER NOT NULL, - CONSTRAINT pk_files PRIMARY KEY (id), - CONSTRAINT fk_files_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_files_uuid UNIQUE (uuid), - CONSTRAINT files_compare_is_downloaded_vs_is_decrypted - CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - CONSTRAINT ck_files_is_downloaded CHECK (is_downloaded IN (0, 1)), - CONSTRAINT ck_files_is_read CHECK (is_read IN (0, 1)), - CONSTRAINT ck_files_is_decrypted CHECK (is_decrypted IN (0, 1)) -); -""" - - -CREATE_TABLE_MESSAGES_NEW = """ - CREATE TABLE messages ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER NOT NULL, - download_url VARCHAR(255) NOT NULL, - is_downloaded BOOLEAN DEFAULT 0 NOT NULL, - is_decrypted BOOLEAN CONSTRAINT messages_compare_is_downloaded_vs_is_decrypted CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - download_error_id INTEGER, - is_read BOOLEAN DEFAULT 0 NOT NULL, - content TEXT CONSTRAINT ck_message_compare_download_vs_content CHECK (CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END), - source_id INTEGER NOT NULL, - last_updated DATETIME NOT NULL, - CONSTRAINT pk_messages PRIMARY KEY (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_messages_uuid UNIQUE (uuid), - CONSTRAINT ck_messages_is_downloaded CHECK (is_downloaded IN (0, 1)), - CONSTRAINT ck_messages_is_decrypted CHECK (is_decrypted IN (0, 1)), - CONSTRAINT fk_messages_download_error_id_downloaderrors FOREIGN KEY(download_error_id) REFERENCES downloaderrors (id), - CONSTRAINT ck_messages_is_read CHECK (is_read IN (0, 1)), - CONSTRAINT fk_messages_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id) - ); -""" - -CREATE_TABLE_MESSAGES_OLD = """ - CREATE TABLE messages ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - source_id INTEGER NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER NOT NULL, - content TEXT, - is_decrypted BOOLEAN, - is_downloaded BOOLEAN DEFAULT 0 NOT NULL, - is_read BOOLEAN DEFAULT 0 NOT NULL, - download_url VARCHAR(255) NOT NULL, - CONSTRAINT pk_messages PRIMARY KEY (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_messages_uuid UNIQUE (uuid), - CONSTRAINT fk_messages_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id), - CONSTRAINT ck_message_compare_download_vs_content - CHECK (CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END), - CONSTRAINT messages_compare_is_downloaded_vs_is_decrypted - CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - CONSTRAINT ck_messages_is_decrypted CHECK (is_decrypted IN (0, 1)), - CONSTRAINT ck_messages_is_downloaded CHECK (is_downloaded IN (0, 1)), - CONSTRAINT ck_messages_is_read CHECK (is_read IN (0, 1)) - ); -""" - - -CREATE_TABLE_REPLIES_NEW = """ - CREATE TABLE replies ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - source_id INTEGER NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER, - content TEXT, - is_decrypted BOOLEAN, - is_downloaded BOOLEAN, - download_error_id INTEGER, - journalist_id INTEGER, - last_updated DATETIME NOT NULL, - CONSTRAINT pk_replies PRIMARY KEY (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_replies_uuid UNIQUE (uuid), - CONSTRAINT fk_replies_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id), - CONSTRAINT fk_replies_download_error_id_downloaderrors - FOREIGN KEY(download_error_id) REFERENCES downloaderrors (id), - CONSTRAINT fk_replies_journalist_id_users FOREIGN KEY(journalist_id) REFERENCES users (id), - CONSTRAINT replies_compare_download_vs_content - CHECK (CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END), - CONSTRAINT replies_compare_is_downloaded_vs_is_decrypted - CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - CONSTRAINT ck_replies_is_decrypted CHECK (is_decrypted IN (0, 1)), - CONSTRAINT ck_replies_is_downloaded CHECK (is_downloaded IN (0, 1)) - ); -""" - -CREATE_TABLE_REPLIES_OLD = """ - CREATE TABLE replies ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - source_id INTEGER NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER, - content TEXT, - is_decrypted BOOLEAN, - is_downloaded BOOLEAN, - journalist_id INTEGER, - CONSTRAINT pk_replies PRIMARY KEY (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_replies_uuid UNIQUE (uuid), - CONSTRAINT fk_replies_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id), - CONSTRAINT fk_replies_journalist_id_users FOREIGN KEY(journalist_id) REFERENCES users (id), - CONSTRAINT replies_compare_download_vs_content - CHECK (CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END), - CONSTRAINT replies_compare_is_downloaded_vs_is_decrypted - CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - CONSTRAINT ck_replies_is_decrypted CHECK (is_decrypted IN (0, 1)), - CONSTRAINT ck_replies_is_downloaded CHECK (is_downloaded IN (0, 1)) - ); -""" - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "downloaderrors", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("name", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_downloaderrors")), - sa.UniqueConstraint("name", name=op.f("uq_downloaderrors_name")), - ) - - conn = op.get_bind() - for name, member in db.DownloadErrorCodes.__members__.items(): - conn.execute("""INSERT INTO downloaderrors (name) VALUES (:name);""", name) - - op.rename_table("files", "files_tmp") - op.rename_table("messages", "messages_tmp") - op.rename_table("replies", "replies_tmp") - - conn.execute(CREATE_TABLE_FILES_NEW) - conn.execute(CREATE_TABLE_MESSAGES_NEW) - conn.execute(CREATE_TABLE_REPLIES_NEW) - - conn.execute( - """ - INSERT INTO files - ( - id, uuid, filename, file_counter, size, download_url, - is_downloaded, is_read, is_decrypted, download_error_id, source_id, - last_updated - ) - SELECT id, uuid, filename, file_counter, size, download_url, - is_downloaded, is_read, is_decrypted, NULL, source_id, CURRENT_TIMESTAMP - FROM files_tmp - """ - ) - - conn.execute( - """ - INSERT INTO messages - ( - id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, is_read, download_error_id, download_url, last_updated - ) - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, is_read, NULL, download_url, CURRENT_TIMESTAMP - FROM messages_tmp - """ - ) - - conn.execute( - """ - INSERT INTO replies - ( - id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, download_error_id, journalist_id, last_updated - ) - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, NULL, journalist_id, CURRENT_TIMESTAMP - FROM replies_tmp - """ - ) - - # Delete the old tables. - op.drop_table("files_tmp") - op.drop_table("messages_tmp") - op.drop_table("replies_tmp") - - # ### end Alembic commands ### - - -def downgrade(): - - conn = op.get_bind() - - op.rename_table("files", "files_tmp") - op.rename_table("messages", "messages_tmp") - op.rename_table("replies", "replies_tmp") - - conn.execute(CREATE_TABLE_FILES_OLD) - conn.execute(CREATE_TABLE_MESSAGES_OLD) - conn.execute(CREATE_TABLE_REPLIES_OLD) - - conn.execute( - """ - INSERT INTO files - (id, uuid, filename, file_counter, size, download_url, - is_downloaded, is_read, is_decrypted, source_id) - SELECT id, uuid, filename, file_counter, size, download_url, - is_downloaded, is_read, is_decrypted, source_id - FROM files_tmp - """ - ) - conn.execute( - """ - INSERT INTO messages - (id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, is_read, download_url) - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, is_read, download_url - FROM messages_tmp - """ - ) - conn.execute( - """ - INSERT INTO replies - (id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, journalist_id) - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, journalist_id - FROM replies_tmp - """ - ) - - # Delete the old tables. - op.drop_table("files_tmp") - op.drop_table("messages_tmp") - op.drop_table("replies_tmp") - - # Drop downloaderrors - op.drop_table("downloaderrors") - - # ### end Alembic commands ### diff --git a/alembic/versions/86b01b6290da_add_reply_draft.py b/alembic/versions/86b01b6290da_add_reply_draft.py deleted file mode 100644 index 77d5f4ba7..000000000 --- a/alembic/versions/86b01b6290da_add_reply_draft.py +++ /dev/null @@ -1,68 +0,0 @@ -"""add reply draft - -Revision ID: 86b01b6290da -Revises: 36a79ffcfbfb -Create Date: 2019-10-17 09:45:07.643076 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "86b01b6290da" -down_revision = "36a79ffcfbfb" -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - "replysendstatuses", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("name", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_replysendstatuses")), - sa.UniqueConstraint("name", name=op.f("uq_replysendstatuses_name")), - ) - - # Set the initial in-progress send statuses: PENDING, FAILED - conn = op.get_bind() - conn.execute( - """ - INSERT INTO replysendstatuses - ('name') - VALUES - ('PENDING'), - ('FAILED'); - """ - ) - - op.create_table( - "draftreplies", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("timestamp", sa.DateTime(), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("send_status_id", sa.Integer(), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_draftreplies")), - sa.UniqueConstraint("uuid", name=op.f("uq_draftreplies_uuid")), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_draftreplies_source_id_sources") - ), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_draftreplies_journalist_id_users") - ), - sa.ForeignKeyConstraint( - ["send_status_id"], - ["replysendstatuses.id"], - op.f("fk_draftreplies_send_status_id_replysendstatuses"), - ), - ) - - -def downgrade(): - op.drop_table("draftreplies") - op.drop_table("replysendstatuses") diff --git a/alembic/versions/9ba8d7524871_add_deletedsource_table.py b/alembic/versions/9ba8d7524871_add_deletedsource_table.py deleted file mode 100644 index cf2aebd92..000000000 --- a/alembic/versions/9ba8d7524871_add_deletedsource_table.py +++ /dev/null @@ -1,32 +0,0 @@ -"""add deletedsource table - -Revision ID: 9ba8d7524871 -Revises: eff1387cfd0b -Create Date: 2022-03-15 12:25:59.145300 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "9ba8d7524871" -down_revision = "eff1387cfd0b" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### Add DeletedSource table ### - op.create_table( - "deletedsource", - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedsource")), - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### Remove DeletedSource table ### - op.drop_table("deletedsource") - # ### end Alembic commands ### diff --git a/alembic/versions/a4bf1f58ce69_fix_journalist_association_in_replies.py b/alembic/versions/a4bf1f58ce69_fix_journalist_association_in_replies.py deleted file mode 100644 index 35bbaecb9..000000000 --- a/alembic/versions/a4bf1f58ce69_fix_journalist_association_in_replies.py +++ /dev/null @@ -1,85 +0,0 @@ -"""fix journalist association in replies table - -Revision ID: a4bf1f58ce69 -Revises: 7f682532afa2 -Create Date: 2020-10-20 13:49:53.035383 - -""" -from alembic import op - -# revision identifiers, used by Alembic. -revision = "a4bf1f58ce69" -down_revision = "7f682532afa2" -branch_labels = None -depends_on = None - - -def upgrade(): - """ - Fix reply association with journalist by updating journalist uuid to journalist id in the - journalist_id column for the replies and draftreplies tables. - """ - conn = op.get_bind() - cursor = conn.execute( - """ - SELECT journalist_id - FROM replies, users - WHERE journalist_id=users.uuid; - """ - ) - - replies_with_incorrect_associations = cursor.fetchall() - if replies_with_incorrect_associations: - conn.execute( - """ - UPDATE replies - SET journalist_id= - ( - SELECT users.id - FROM users - WHERE journalist_id=users.uuid - ) - WHERE exists - ( - SELECT users.id - FROM users - WHERE journalist_id=users.uuid - ); - """ - ) - - cursor = conn.execute( - """ - SELECT journalist_id - FROM draftreplies, users - WHERE journalist_id=users.uuid; - """ - ) - - draftreplies_with_incorrect_associations = cursor.fetchall() - if draftreplies_with_incorrect_associations: - conn.execute( - """ - UPDATE draftreplies - SET journalist_id= - ( - SELECT users.id - FROM users - WHERE journalist_id=users.uuid - ) - WHERE exists - ( - SELECT users.id - FROM users - WHERE journalist_id=users.uuid - ); - """ - ) - - -def downgrade(): - """ - We do not want to undo this bug fix, because nothing will break if we downgrade to an earlier - version of the client. - """ - pass diff --git a/alembic/versions/bafdcae12f97_.py b/alembic/versions/bafdcae12f97_.py deleted file mode 100644 index 9e9408a78..000000000 --- a/alembic/versions/bafdcae12f97_.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Add files.original_filename - -Revision ID: bafdcae12f97 -Revises: fecf1191b6f0 -Create Date: 2019-06-24 13:45:47.239212 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "bafdcae12f97" -down_revision = "fecf1191b6f0" -branch_labels = None -depends_on = None - - -def upgrade(): - op.add_column( - "files", - sa.Column("original_filename", sa.String(length=255), nullable=False, server_default=""), - ) - - conn = op.get_bind() - conn.execute("""UPDATE files SET original_filename = replace(filename, '.gz.gpg', '');""") - - -def downgrade(): - with op.batch_alter_table("files", schema=None) as batch_op: - batch_op.drop_column("original_filename") diff --git a/alembic/versions/bd57477f19a2_add_seen_tables.py b/alembic/versions/bd57477f19a2_add_seen_tables.py deleted file mode 100644 index 05809069f..000000000 --- a/alembic/versions/bd57477f19a2_add_seen_tables.py +++ /dev/null @@ -1,78 +0,0 @@ -"""add seen tables - -Revision ID: bd57477f19a2 -Revises: a4bf1f58ce69 -Create Date: 2020-10-20 22:43:46.743035 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "bd57477f19a2" -down_revision = "a4bf1f58ce69" -branch_labels = None -depends_on = None - - -def upgrade(): - """ - Create seen tables for files, messages, and replies. - - Once freedomofpress/securedrop#5503 is fixed, we can expect that journalist_id will never be - NULL and then migrate these tables so that both of the foreign keys make up the primary key. - We will also need to migrate any existing NULL journalist IDs in these tables (as well as the - 'replies' table) to a non-NULL id, most likely to the ID of a special global "deleted" User - only to be used for historical data. - """ - op.create_table( - "seen_files", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("file_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.ForeignKeyConstraint( - ["file_id"], ["files.id"], name=op.f("fk_seen_files_file_id_files") - ), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_seen_files_journalist_id_users") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_files")), - sa.UniqueConstraint("file_id", "journalist_id", name=op.f("uq_seen_files_file_id")), - ) - op.create_table( - "seen_messages", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("message_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_seen_messages_journalist_id_users") - ), - sa.ForeignKeyConstraint( - ["message_id"], ["messages.id"], name=op.f("fk_seen_messages_message_id_messages") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_messages")), - sa.UniqueConstraint( - "message_id", "journalist_id", name=op.f("uq_seen_messages_message_id") - ), - ) - op.create_table( - "seen_replies", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("reply_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_seen_replies_journalist_id_users") - ), - sa.ForeignKeyConstraint( - ["reply_id"], ["replies.id"], name=op.f("fk_seen_replies_reply_id_replies") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_replies")), - sa.UniqueConstraint("reply_id", "journalist_id", name=op.f("uq_seen_replies_reply_id")), - ) - - -def downgrade(): - op.drop_table("seen_replies") - op.drop_table("seen_messages") - op.drop_table("seen_files") diff --git a/alembic/versions/d7c8af95bc8e_initial.py b/alembic/versions/d7c8af95bc8e_initial.py new file mode 100644 index 000000000..eb89d0785 --- /dev/null +++ b/alembic/versions/d7c8af95bc8e_initial.py @@ -0,0 +1,256 @@ +"""Initial schema, squashed from old head 9ba8d7524871 (per #1500). + +Revision ID: d7c8af95bc8e +Revises: +Create Date: 2022-06-09 16:41:11.913336 + +""" +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d7c8af95bc8e" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "deletedconversation", + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedconversation")), + ) + op.create_table( + "deletedsource", + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedsource")), + ) + op.create_table( + "downloaderrors", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_downloaderrors")), + sa.UniqueConstraint("name", name=op.f("uq_downloaderrors_name")), + ) + op.create_table( + "replysendstatuses", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_replysendstatuses")), + sa.UniqueConstraint("name", name=op.f("uq_replysendstatuses_name")), + ) + op.create_table( + "sources", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("journalist_designation", sa.String(length=255), nullable=False), + sa.Column("document_count", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column( + "is_flagged", sa.Boolean(name="is_flagged"), server_default=sa.text("0"), nullable=True + ), + sa.Column("public_key", sa.Text(), nullable=True), + sa.Column("fingerprint", sa.String(length=64), nullable=True), + sa.Column("interaction_count", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column( + "is_starred", sa.Boolean(name="is_starred"), server_default=sa.text("0"), nullable=True + ), + sa.Column("last_updated", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_sources")), + sa.UniqueConstraint("uuid", name=op.f("uq_sources_uuid")), + ) + op.create_table( + "users", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("username", sa.String(length=255), nullable=False), + sa.Column("firstname", sa.String(length=64), nullable=True), + sa.Column("lastname", sa.String(length=64), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_users")), + sa.UniqueConstraint("uuid", name=op.f("uq_users_uuid")), + ) + op.create_table( + "draftreplies", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("timestamp", sa.DateTime(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("journalist_id", sa.Integer(), nullable=True), + sa.Column("file_counter", sa.Integer(), nullable=False), + sa.Column("content", sa.Text(), nullable=True), + sa.Column("send_status_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["journalist_id"], ["users.id"], name=op.f("fk_draftreplies_journalist_id_users") + ), + sa.ForeignKeyConstraint( + ["send_status_id"], + ["replysendstatuses.id"], + name=op.f("fk_draftreplies_send_status_id_replysendstatuses"), + ), + sa.ForeignKeyConstraint( + ["source_id"], ["sources.id"], name=op.f("fk_draftreplies_source_id_sources") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_draftreplies")), + sa.UniqueConstraint("uuid", name=op.f("uq_draftreplies_uuid")), + ) + op.create_table( + "files", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("file_counter", sa.Integer(), nullable=False), + sa.Column("size", sa.Integer(), nullable=False), + sa.Column("download_url", sa.String(length=255), nullable=False), + sa.Column( + "is_downloaded", + sa.Boolean(name="is_downloaded"), + server_default=sa.text("0"), + nullable=False, + ), + sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), + sa.Column("download_error_id", sa.Integer(), nullable=True), + sa.Column( + "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False + ), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("last_updated", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["download_error_id"], + ["downloaderrors.id"], + name=op.f("fk_files_download_error_id_downloaderrors"), + ), + sa.ForeignKeyConstraint( + ["source_id"], ["sources.id"], name=op.f("fk_files_source_id_sources") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_files")), + sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), + sa.UniqueConstraint("uuid", name=op.f("uq_files_uuid")), + ) + op.create_table( + "messages", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("file_counter", sa.Integer(), nullable=False), + sa.Column("size", sa.Integer(), nullable=False), + sa.Column("download_url", sa.String(length=255), nullable=False), + sa.Column( + "is_downloaded", + sa.Boolean(name="is_downloaded"), + server_default=sa.text("0"), + nullable=False, + ), + sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), + sa.Column("download_error_id", sa.Integer(), nullable=True), + sa.Column( + "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False + ), + sa.Column("content", sa.Text(), nullable=True), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("last_updated", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["download_error_id"], + ["downloaderrors.id"], + name=op.f("fk_messages_download_error_id_downloaderrors"), + ), + sa.ForeignKeyConstraint( + ["source_id"], ["sources.id"], name=op.f("fk_messages_source_id_sources") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), + sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), + sa.UniqueConstraint("uuid", name=op.f("uq_messages_uuid")), + ) + op.create_table( + "replies", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("journalist_id", sa.Integer(), nullable=True), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("file_counter", sa.Integer(), nullable=False), + sa.Column("size", sa.Integer(), nullable=True), + sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), + sa.Column("content", sa.Text(), nullable=True), + sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), + sa.Column("download_error_id", sa.Integer(), nullable=True), + sa.Column("last_updated", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["download_error_id"], + ["downloaderrors.id"], + name=op.f("fk_replies_download_error_id_downloaderrors"), + ), + sa.ForeignKeyConstraint( + ["journalist_id"], ["users.id"], name=op.f("fk_replies_journalist_id_users") + ), + sa.ForeignKeyConstraint( + ["source_id"], ["sources.id"], name=op.f("fk_replies_source_id_sources") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_replies")), + sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), + sa.UniqueConstraint("uuid", name=op.f("uq_replies_uuid")), + ) + op.create_table( + "seen_files", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("file_id", sa.Integer(), nullable=False), + sa.Column("journalist_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["file_id"], ["files.id"], name=op.f("fk_seen_files_file_id_files") + ), + sa.ForeignKeyConstraint( + ["journalist_id"], ["users.id"], name=op.f("fk_seen_files_journalist_id_users") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_files")), + sa.UniqueConstraint("file_id", "journalist_id", name=op.f("uq_seen_files_file_id")), + ) + op.create_table( + "seen_messages", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("message_id", sa.Integer(), nullable=False), + sa.Column("journalist_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["journalist_id"], ["users.id"], name=op.f("fk_seen_messages_journalist_id_users") + ), + sa.ForeignKeyConstraint( + ["message_id"], ["messages.id"], name=op.f("fk_seen_messages_message_id_messages") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_messages")), + sa.UniqueConstraint( + "message_id", "journalist_id", name=op.f("uq_seen_messages_message_id") + ), + ) + op.create_table( + "seen_replies", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("reply_id", sa.Integer(), nullable=False), + sa.Column("journalist_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["journalist_id"], ["users.id"], name=op.f("fk_seen_replies_journalist_id_users") + ), + sa.ForeignKeyConstraint( + ["reply_id"], ["replies.id"], name=op.f("fk_seen_replies_reply_id_replies") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_replies")), + sa.UniqueConstraint("reply_id", "journalist_id", name=op.f("uq_seen_replies_reply_id")), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("seen_replies") + op.drop_table("seen_messages") + op.drop_table("seen_files") + op.drop_table("replies") + op.drop_table("messages") + op.drop_table("files") + op.drop_table("draftreplies") + op.drop_table("users") + op.drop_table("sources") + op.drop_table("replysendstatuses") + op.drop_table("downloaderrors") + op.drop_table("deletedsource") + op.drop_table("deletedconversation") + # ### end Alembic commands ### diff --git a/alembic/versions/eff1387cfd0b_add_deletedconversation_table.py b/alembic/versions/eff1387cfd0b_add_deletedconversation_table.py deleted file mode 100644 index 0ad578e25..000000000 --- a/alembic/versions/eff1387cfd0b_add_deletedconversation_table.py +++ /dev/null @@ -1,34 +0,0 @@ -"""add deletedconversation table - -Revision ID: eff1387cfd0b -Revises: bd57477f19a2 -Create Date: 2022-02-24 13:11:22.227528 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "eff1387cfd0b" -down_revision = "bd57477f19a2" -branch_labels = None -depends_on = None - - -def upgrade(): - # Add deletedconversation table to manage locally-deleted records - # and ensure they do not get re-downloaded to the database during - # a network race condition. - # UUID was chosen as PK to avoid storing data such as source_id that - # could divulge information about the source account creation timeline. - # Note that records in this table are purged every 15 seconds. - op.create_table( - "deletedconversation", - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedconversation")), - ) - - -def downgrade(): - op.drop_table("deletedconversation") diff --git a/alembic/versions/fb657f2ee8a7_drop_file_original_filename.py b/alembic/versions/fb657f2ee8a7_drop_file_original_filename.py deleted file mode 100644 index a09c7533c..000000000 --- a/alembic/versions/fb657f2ee8a7_drop_file_original_filename.py +++ /dev/null @@ -1,72 +0,0 @@ -"""drop File.original_filename - -Revision ID: fb657f2ee8a7 -Revises: 86b01b6290da -Create Date: 2020-01-23 18:55:09.857324 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "fb657f2ee8a7" -down_revision = "86b01b6290da" -branch_labels = None -depends_on = None - - -def upgrade(): - conn = op.get_bind() - - op.rename_table("files", "original_files") - - conn.execute( - """ - CREATE TABLE files ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER NOT NULL, - download_url VARCHAR(255) NOT NULL, - is_downloaded BOOLEAN DEFAULT 0 NOT NULL, - is_read BOOLEAN DEFAULT 0 NOT NULL, - is_decrypted BOOLEAN, - source_id INTEGER NOT NULL, - CONSTRAINT pk_files PRIMARY KEY (id), - CONSTRAINT fk_files_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_files_uuid UNIQUE (uuid), - CONSTRAINT files_compare_is_downloaded_vs_is_decrypted CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - CONSTRAINT ck_files_is_downloaded CHECK (is_downloaded IN (0, 1)), - CONSTRAINT ck_files_is_read CHECK (is_read IN (0, 1)), - CONSTRAINT ck_files_is_decrypted CHECK (is_decrypted IN (0, 1)) - ) - """ - ) - - conn.execute( - """ - INSERT INTO files - (id, uuid, filename, file_counter, size, download_url, is_downloaded, - is_decrypted, is_read, source_id) - SELECT id, uuid, filename, file_counter, size, download_url, is_downloaded, - is_decrypted, is_read, source_id - FROM original_files - """ - ) - - op.drop_table("original_files") - - -def downgrade(): - op.add_column( - "files", - sa.Column( - "original_filename", - sa.VARCHAR(length=255), - server_default=sa.text("''"), - nullable=False, - ), - ) diff --git a/alembic/versions/fecf1191b6f0_remove_decryption_vs_content_contraint.py b/alembic/versions/fecf1191b6f0_remove_decryption_vs_content_contraint.py deleted file mode 100644 index f611da5ae..000000000 --- a/alembic/versions/fecf1191b6f0_remove_decryption_vs_content_contraint.py +++ /dev/null @@ -1,218 +0,0 @@ -"""remove decryption_vs_content contraint - -Revision ID: fecf1191b6f0 -Revises: 2f363b3d680e -Create Date: 2019-06-18 19:14:17.862910 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "fecf1191b6f0" -down_revision = "2f363b3d680e" -branch_labels = None -depends_on = None - - -def upgrade(): - # Save existing tables we want to modify. - op.rename_table("messages", "messages_tmp") - op.rename_table("replies", "replies_tmp") - - # Create new tables without the constraint. - op.create_table( - "messages", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=False), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column( - "is_downloaded", - sa.Boolean(name="is_downloaded"), - server_default=sa.text("0"), - nullable=False, - ), - sa.Column( - "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False - ), - sa.Column("download_url", sa.String(length=255), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_messages_uuid")), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_messages_source_id_sources") - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name=op.f("ck_message_compare_download_vs_content"), - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="messages_compare_is_downloaded_vs_is_decrypted", - ), - ) - - op.create_table( - "replies", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=True), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_replies")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_replies_uuid")), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_replies_source_id_sources") - ), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_replies_journalist_id_users") - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name="replies_compare_download_vs_content", - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="replies_compare_is_downloaded_vs_is_decrypted", - ), - ) - - # Copy existing data into new tables. - conn = op.get_bind() - conn.execute( - """ - INSERT INTO messages - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, is_read, download_url - FROM messages_tmp - """ - ) - conn.execute( - """ - INSERT INTO replies - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, journalist_id - FROM replies_tmp - """ - ) - - # Delete the old tables. - op.drop_table("messages_tmp") - op.drop_table("replies_tmp") - - -def downgrade(): - # Save existing tables we want to modify. - op.rename_table("messages", "messages_tmp") - op.rename_table("replies", "replies_tmp") - - # Create new tables with the constraint. - op.create_table( - "messages", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=False), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column( - "is_downloaded", - sa.Boolean(name="is_downloaded"), - server_default=sa.text("0"), - nullable=False, - ), - sa.Column( - "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False - ), - sa.Column("download_url", sa.String(length=255), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_messages_uuid")), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_messages_source_id_sources") - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name=op.f("ck_message_compare_download_vs_content"), - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="messages_compare_is_downloaded_vs_is_decrypted", - ), - sa.CheckConstraint( - "CASE WHEN is_decrypted = 0 THEN content IS NULL ELSE 1 END", - name="messages_compare_is_decrypted_vs_content", - ), - ) - - op.create_table( - "replies", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=True), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_replies")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_replies_uuid")), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_replies_source_id_sources") - ), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_replies_journalist_id_users") - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name="replies_compare_download_vs_content", - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="replies_compare_is_downloaded_vs_is_decrypted", - ), - sa.CheckConstraint( - "CASE WHEN is_decrypted = 0 THEN content IS NULL ELSE 1 END", - name="replies_compare_is_decrypted_vs_content", - ), - ) - - # Copy existing data into new tables. - conn = op.get_bind() - conn.execute( - """ - INSERT INTO messages - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, download_url, is_read - FROM messages_tmp - """ - ) - conn.execute( - """ - INSERT INTO replies - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, journalist_id - FROM replies_tmp - """ - ) - - # Delete the old tables. - op.drop_table("messages_tmp") - op.drop_table("replies_tmp") From ec43eca26edbd341a4a21ba86e84078430a89e9d Mon Sep 17 00:00:00 2001 From: Cory Francis Myers Date: Mon, 13 Jun 2022 18:17:04 -0700 Subject: [PATCH 02/10] fix: restore column constraints missed by "alembic revision --autogenerate" --- alembic/versions/d7c8af95bc8e_initial.py | 50 +++++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/alembic/versions/d7c8af95bc8e_initial.py b/alembic/versions/d7c8af95bc8e_initial.py index eb89d0785..f41334cf0 100644 --- a/alembic/versions/d7c8af95bc8e_initial.py +++ b/alembic/versions/d7c8af95bc8e_initial.py @@ -109,7 +109,15 @@ def upgrade(): server_default=sa.text("0"), nullable=False, ), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), + sa.Column( + "is_decrypted", + sa.Boolean(name="is_decrypted"), + sa.CheckConstraint( + "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", + name="files_compare_is_downloaded_vs_is_decrypted", + ), + nullable=True, + ), sa.Column("download_error_id", sa.Integer(), nullable=True), sa.Column( "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False @@ -142,12 +150,28 @@ def upgrade(): server_default=sa.text("0"), nullable=False, ), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), + sa.Column( + "is_decrypted", + sa.Boolean(name="is_decrypted"), + sa.CheckConstraint( + "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", + name="messages_compare_is_downloaded_vs_is_decrypted", + ), + nullable=True, + ), sa.Column("download_error_id", sa.Integer(), nullable=True), sa.Column( "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False ), - sa.Column("content", sa.Text(), nullable=True), + sa.Column( + "content", + sa.Text(), + sa.CheckConstraint( + "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", + name="ck_message_compare_download_vs_content", + ), + nullable=True, + ), sa.Column("source_id", sa.Integer(), nullable=False), sa.Column("last_updated", sa.DateTime(), nullable=False), sa.ForeignKeyConstraint( @@ -172,8 +196,24 @@ def upgrade(): sa.Column("file_counter", sa.Integer(), nullable=False), sa.Column("size", sa.Integer(), nullable=True), sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), + sa.Column( + "content", + sa.Text(), + sa.CheckConstraint( + "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", + name="replies_compare_download_vs_content", + ), + nullable=True, + ), + sa.Column( + "is_decrypted", + sa.Boolean(name="is_decrypted"), + sa.CheckConstraint( + "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", + name="replies_compare_is_downloaded_vs_is_decrypted", + ), + nullable=True, + ), sa.Column("download_error_id", sa.Integer(), nullable=True), sa.Column("last_updated", sa.DateTime(), nullable=False), sa.ForeignKeyConstraint( From cce42541aefdf4bba2f346084d5d69bd8dd1bd26 Mon Sep 17 00:00:00 2001 From: Cory Francis Myers Date: Thu, 9 Jun 2022 16:49:01 -0700 Subject: [PATCH 03/10] test: remove data-migration tests for deleted Alembic versions Data-migration tests MUST not use models from securedrop_client.db, which will always contain the definitions from Git's head, not those corresponding to the Alembic head under test. --- tests/migrations/test_a4bf1f58ce69.py | 105 ------------ tests/migrations/test_bd57477f19a2.py | 235 -------------------------- tests/test_alembic.py | 2 +- 3 files changed, 1 insertion(+), 341 deletions(-) delete mode 100644 tests/migrations/test_a4bf1f58ce69.py delete mode 100644 tests/migrations/test_bd57477f19a2.py diff --git a/tests/migrations/test_a4bf1f58ce69.py b/tests/migrations/test_a4bf1f58ce69.py deleted file mode 100644 index 84c60d09a..000000000 --- a/tests/migrations/test_a4bf1f58ce69.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import random -import subprocess - -from securedrop_client import db -from securedrop_client.db import DraftReply, Reply, User - -from .utils import add_draft_reply, add_reply, add_source, add_user - -random.seed("=^..^=..^=..^=") - - -class UpgradeTester: - """ - Verify that upgrading to the target migration results in the replacement of uuid with id of the - user in the replies table's journalist_id column. - """ - - NUM_USERS = 20 - NUM_SOURCES = 20 - NUM_REPLIES = 40 - - def __init__(self, homedir): - subprocess.check_call(["sqlite3", os.path.join(homedir, "svs.sqlite"), ".databases"]) - self.session = db.make_session_maker(homedir)() - - def load_data(self): - """ - Load data that has the bug where user.uuid is stored in replies.journalist_id and - draftreplies.journalist_id. - """ - for _ in range(self.NUM_SOURCES): - add_source(self.session) - - self.session.commit() - - for i in range(self.NUM_USERS): - if i == 0: - # As of this migration, the server tells the client that the associated journalist - # of a reply has been deleted by returning "deleted" as the uuid of the associated - # journalist. This gets stored as the jouranlist_id in the replies table. - # - # Make sure to test this case as well. - add_user(self.session, "deleted") - source_id = random.randint(1, self.NUM_SOURCES) - add_reply(self.session, "deleted", source_id) - else: - add_user(self.session) - - self.session.commit() - - # Add replies from randomly-selected journalists to a randomly-selected sources - for _ in range(1, self.NUM_REPLIES): - journalist_id = random.randint(1, self.NUM_USERS) - journalist = self.session.query(User).filter_by(id=journalist_id).one() - source_id = random.randint(1, self.NUM_SOURCES) - add_reply(self.session, journalist.uuid, source_id) - - # Add draft replies from randomly-selected journalists to a randomly-selected sources - for _ in range(1, self.NUM_REPLIES): - journalist_id = random.randint(1, self.NUM_USERS) - journalist = self.session.query(User).filter_by(id=journalist_id).one() - source_id = random.randint(1, self.NUM_SOURCES) - add_draft_reply(self.session, journalist.uuid, source_id) - - self.session.commit() - - def check_upgrade(self): - """ - Make sure each reply in the replies and draftreplies tables have the correct journalist_id - stored for the associated journalist by making sure a User account exists with that - journalist id. - """ - replies = self.session.query(Reply).all() - assert len(replies) - - for reply in replies: - # Will fail if User does not exist - self.session.query(User).filter_by(id=reply.journalist_id).one() - - draftreplies = self.session.query(DraftReply).all() - assert len(draftreplies) - - for draftreply in draftreplies: - # Will fail if User does not exist - self.session.query(User).filter_by(id=draftreply.journalist_id).one() - - self.session.close() - - -class DowngradeTester: - """ - Nothing to test since the downgrade path doesn't do anything. - """ - - def __init__(self, homedir): - pass - - def load_data(self): - pass - - def check_downgrade(self): - pass diff --git a/tests/migrations/test_bd57477f19a2.py b/tests/migrations/test_bd57477f19a2.py deleted file mode 100644 index 0bb3b0229..000000000 --- a/tests/migrations/test_bd57477f19a2.py +++ /dev/null @@ -1,235 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import random -import subprocess - -import pytest -from sqlalchemy import text -from sqlalchemy.exc import IntegrityError - -from securedrop_client import db -from securedrop_client.db import Reply, User - -from .utils import ( - add_file, - add_message, - add_reply, - add_source, - add_user, - mark_file_as_seen, - mark_message_as_seen, - mark_reply_as_seen, -) - - -class UpgradeTester: - """ - Verify that upgrading to the target migration results in the creation of the seen tables. - """ - - NUM_USERS = 20 - NUM_SOURCES = 20 - NUM_REPLIES = 40 - - def __init__(self, homedir): - subprocess.check_call(["sqlite3", os.path.join(homedir, "svs.sqlite"), ".databases"]) - self.session = db.make_session_maker(homedir)() - - def load_data(self): - for source_id in range(1, self.NUM_SOURCES + 1): - add_source(self.session) - - # Add zero to a few messages from each source, some messages are set to downloaded - for _ in range(random.randint(0, 2)): - add_message(self.session, source_id) - - # Add zero to a few files from each source, some files are set to downloaded - for _ in range(random.randint(0, 2)): - add_file(self.session, source_id) - - self.session.commit() - - for i in range(self.NUM_USERS): - if i == 0: - # As of this migration, the server tells the client that the associated journalist - # of a reply has been deleted by returning "deleted" as the uuid of the associated - # journalist. This gets stored as the jouranlist_id in the replies table. - # - # Make sure to test this case as well. - add_user(self.session, "deleted") - source_id = random.randint(1, self.NUM_SOURCES) - user = self.session.query(User).filter_by(uuid="deleted").one() - add_reply(self.session, user.id, source_id) - else: - add_user(self.session) - - self.session.commit() - - # Add replies from randomly-selected journalists to a randomly-selected sources - for _ in range(1, self.NUM_REPLIES): - journalist_id = random.randint(1, self.NUM_USERS) - source_id = random.randint(1, self.NUM_SOURCES) - add_reply(self.session, journalist_id, source_id) - - self.session.commit() - - def check_upgrade(self): - """ - Make sure seen tables exist and work as expected. - """ - replies = self.session.query(Reply).all() - assert len(replies) - - for reply in replies: - # Will fail if User does not exist - self.session.query(User).filter_by(id=reply.journalist_id).one() - - sql = "SELECT * FROM files" - files = self.session.execute(text(sql)).fetchall() - - sql = "SELECT * FROM messages" - messages = self.session.execute(text(sql)).fetchall() - - sql = "SELECT * FROM replies" - replies = self.session.execute(text(sql)).fetchall() - - # Now seen tables exist, so you should be able to mark some files, messages, and replies - # as seen - for file in files: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_file_as_seen(self.session, file.id, selected_journo_id) - for message in messages: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_message_as_seen(self.session, message.id, selected_journo_id) - for reply in replies: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_reply_as_seen(self.session, reply.id, selected_journo_id) - - # Check unique constraint on (reply_id, journalist_id) - params = {"reply_id": 100, "journalist_id": 100} - sql = """ - INSERT INTO seen_replies (reply_id, journalist_id) - VALUES (:reply_id, :journalist_id); - """ - self.session.execute(text(sql), params) - with pytest.raises(IntegrityError): - self.session.execute(text(sql), params) - - # Check unique constraint on (message_id, journalist_id) - params = {"message_id": 100, "journalist_id": 100} - sql = """ - INSERT INTO seen_messages (message_id, journalist_id) - VALUES (:message_id, :journalist_id); - """ - self.session.execute(text(sql), params) - with pytest.raises(IntegrityError): - self.session.execute(text(sql), params) - - # Check unique constraint on (file_id, journalist_id) - params = {"file_id": 101, "journalist_id": 100} - sql = """ - INSERT INTO seen_files (file_id, journalist_id) - VALUES (:file_id, :journalist_id); - """ - self.session.execute(text(sql), params) - with pytest.raises(IntegrityError): - self.session.execute(text(sql), params) - - -class DowngradeTester: - """ - Verify that downgrading from the target migration keeps in place the updates from the migration - since there is no need to add bad data back into the db (the migration is backwards compatible). - """ - - NUM_USERS = 20 - NUM_SOURCES = 20 - NUM_REPLIES = 40 - - def __init__(self, homedir): - subprocess.check_call(["sqlite3", os.path.join(homedir, "svs.sqlite"), ".databases"]) - self.session = db.make_session_maker(homedir)() - - def load_data(self): - for source_id in range(1, self.NUM_SOURCES + 1): - add_source(self.session) - - # Add zero to a few messages from each source, some messages are set to downloaded - for _ in range(random.randint(0, 3)): - add_message(self.session, source_id) - - # Add zero to a few files from each source, some files are set to downloaded - for _ in range(random.randint(0, 3)): - add_file(self.session, source_id) - - self.session.commit() - - for i in range(self.NUM_USERS): - if i == 0: - # As of this migration, the server tells the client that the associated journalist - # of a reply has been deleted by returning "deleted" as the uuid of the associated - # journalist. This gets stored as the jouranlist_id in the replies table. - # - # Make sure to test this case as well. - add_user(self.session, "deleted") - source_id = random.randint(1, self.NUM_SOURCES) - add_reply(self.session, "deleted", source_id) - else: - add_user(self.session) - - self.session.commit() - - # Add replies from randomly-selected journalists to a randomly-selected sources - for _ in range(1, self.NUM_REPLIES): - journalist_id = random.randint(1, self.NUM_USERS) - source_id = random.randint(1, self.NUM_SOURCES) - add_reply(self.session, journalist_id, source_id) - - self.session.commit() - - # Mark some files, messages, and replies as seen - sql = "SELECT * FROM files" - files = self.session.execute(text(sql)).fetchall() - for file in files: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_file_as_seen(self.session, file.id, selected_journo_id) - - sql = "SELECT * FROM messages" - messages = self.session.execute(text(sql)).fetchall() - for message in messages: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_message_as_seen(self.session, message.id, selected_journo_id) - - sql = "SELECT * FROM replies" - replies = self.session.execute(text(sql)).fetchall() - for reply in replies: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_reply_as_seen(self.session, reply.id, selected_journo_id) - - self.session.commit() - - def check_downgrade(self): - """ - Check that seen tables no longer exist. - """ - params = {"table_name": "seen_files"} - sql = "SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;" - seen_files_exists = self.session.execute(text(sql), params).fetchall() - assert not seen_files_exists - - params = {"table_name": "seen_messages"} - sql = "SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;" - seen_messages_exists = self.session.execute(text(sql), params).fetchall() - assert not seen_messages_exists - - params = {"table_name": "seen_replies"} - sql = "SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;" - seen_replies_exists = self.session.execute(text(sql), params).fetchall() - assert not seen_replies_exists diff --git a/tests/test_alembic.py b/tests/test_alembic.py index d3a8cdc4f..b165a5c4c 100644 --- a/tests/test_alembic.py +++ b/tests/test_alembic.py @@ -20,7 +20,7 @@ x.split(".")[0].split("_")[0] for x in os.listdir(MIGRATION_PATH) if x.endswith(".py") ] -DATA_MIGRATIONS = ["a4bf1f58ce69", "bd57477f19a2"] +DATA_MIGRATIONS = [] WHITESPACE_REGEX = re.compile(r"\s+") From 0e94279162f85dbc2d32e7352e26548954fc7492 Mon Sep 17 00:00:00 2001 From: Cory Francis Myers Date: Thu, 9 Jun 2022 17:03:09 -0700 Subject: [PATCH 04/10] test: remove version-insensitive utilities for testing data migrations Data-migration tests MUST be self-contained in order to test the schema as of the Alembic version under test, not the current Git head. --- tests/migrations/utils.py | 327 -------------------------------------- 1 file changed, 327 deletions(-) delete mode 100644 tests/migrations/utils.py diff --git a/tests/migrations/utils.py b/tests/migrations/utils.py deleted file mode 100644 index dc89bdafa..000000000 --- a/tests/migrations/utils.py +++ /dev/null @@ -1,327 +0,0 @@ -# -*- coding: utf-8 -*- -import random -import string -from datetime import datetime -from typing import Optional -from uuid import uuid4 - -from sqlalchemy import text -from sqlalchemy.orm.session import Session - -from securedrop_client.db import DownloadError, ReplySendStatus, Source - -random.seed("ᕕ( ᐛ )ᕗ") - - -def random_bool() -> bool: - return bool(random.getrandbits(1)) - - -def bool_or_none() -> Optional[bool]: - return random.choice([None, True, False]) - - -def random_name() -> str: - len = random.randint(1, 100) - return random_chars(len) - - -def random_username() -> str: - len = random.randint(3, 64) - return random_chars(len) - - -def random_chars(len: int, chars: str = string.printable) -> str: - return "".join([random.choice(chars) for _ in range(len)]) - - -def random_ascii_chars(len: int, chars: str = string.ascii_lowercase): - return "".join([random.choice(chars) for _ in range(len)]) - - -def random_datetime(nullable: bool = False): - if nullable and random_bool(): - return None - else: - return datetime( - year=random.randint(1, 9999), - month=random.randint(1, 12), - day=random.randint(1, 28), - hour=random.randint(0, 23), - minute=random.randint(0, 59), - second=random.randint(0, 59), - microsecond=random.randint(0, 1000), - ) - - -def add_source(session: Session) -> None: - params = { - "uuid": str(uuid4()), - "journalist_designation": random_chars(50), - "last_updated": random_datetime(nullable=True), - "interaction_count": random.randint(0, 1000), - } - sql = """ - INSERT INTO sources ( - uuid, - journalist_designation, - last_updated, - interaction_count - ) - VALUES ( - :uuid, - :journalist_designation, - :last_updated, - :interaction_count - ) - """ - session.execute(text(sql), params) - - -def add_user(session: Session, uuid: Optional[str] = None) -> None: - if not uuid: - journalist_uuid = str(uuid4()) - else: - journalist_uuid = uuid - - params = {"uuid": journalist_uuid, "username": random_username()} - sql = """ - INSERT INTO users (uuid, username) - VALUES (:uuid, :username) - """ - session.execute(text(sql), params) - - -def add_file(session: Session, source_id: int) -> None: - is_downloaded = random_bool() - is_decrypted = random_bool() if is_downloaded else None - - source = session.query(Source).filter_by(id=source_id).one() - file_counter = len(source.collection) + 1 - - params = { - "uuid": str(uuid4()), - "source_id": source_id, - "filename": random_chars(50) + "-doc.gz.gpg", - "file_counter": file_counter, - "size": random.randint(0, 1024 * 1024 * 500), - "download_url": random_chars(50), - "is_downloaded": is_downloaded, - "is_decrypted": is_decrypted, - "is_read": random.choice([True, False]), - "last_updated": random_datetime(), - } - sql = """ - INSERT INTO files - ( - uuid, - source_id, - filename, - file_counter, - size, - download_url, - is_downloaded, - is_decrypted, - is_read, - last_updated - ) - VALUES - ( - :uuid, - :source_id, - :filename, - :file_counter, - :size, - :download_url, - :is_downloaded, - :is_decrypted, - :is_read, - :last_updated - ) - """ - session.execute(text(sql), params) - - -def add_message(session: Session, source_id: int) -> None: - is_downloaded = random_bool() - is_decrypted = random_bool() if is_downloaded else None - - content = random_chars(1000) if is_downloaded else None - - source = session.query(Source).filter_by(id=source_id).one() - file_counter = len(source.collection) + 1 - - params = { - "uuid": str(uuid4()), - "source_id": source_id, - "filename": random_chars(50) + "-doc.gz.gpg", - "file_counter": file_counter, - "size": random.randint(0, 1024 * 1024 * 500), - "content": content, - "download_url": random_chars(50), - "is_downloaded": is_downloaded, - "is_decrypted": is_decrypted, - "is_read": random.choice([True, False]), - "last_updated": random_datetime(), - } - sql = """ - INSERT INTO messages - ( - uuid, - source_id, - filename, - file_counter, - size, - content, - download_url, - is_downloaded, - is_decrypted, - is_read, - last_updated - ) - VALUES - ( - :uuid, - :source_id, - :filename, - :file_counter, - :size, - :content, - :download_url, - :is_downloaded, - :is_decrypted, - :is_read, - :last_updated - ) - """ - session.execute(text(sql), params) - - -def add_reply(session: Session, journalist_id: int, source_id: int) -> None: - is_downloaded = random_bool() if random_bool() else None - is_decrypted = random_bool() if is_downloaded else None - - download_errors = session.query(DownloadError).all() - download_error_ids = [error.id for error in download_errors] - - content = random_chars(1000) if is_downloaded else None - - source = session.query(Source).filter_by(id=source_id).one() - file_counter = len(source.collection) + 1 - - params = { - "uuid": str(uuid4()), - "journalist_id": journalist_id, - "source_id": source_id, - "filename": random_chars(50) + "-reply.gpg", - "file_counter": file_counter, - "size": random.randint(0, 1024 * 1024 * 500), - "content": content, - "is_downloaded": is_downloaded, - "is_decrypted": is_decrypted, - "download_error_id": random.choice(download_error_ids), - "last_updated": random_datetime(), - } - sql = """ - INSERT INTO replies - ( - uuid, - journalist_id, - source_id, - filename, - file_counter, - size, - content, - is_downloaded, - is_decrypted, - download_error_id, - last_updated - ) - VALUES - ( - :uuid, - :journalist_id, - :source_id, - :filename, - :file_counter, - :size, - :content, - :is_downloaded, - :is_decrypted, - :download_error_id, - :last_updated - ) - """ - session.execute(text(sql), params) - - -def mark_file_as_seen(session: Session, file_id: int, journalist_id: int) -> None: - params = {"file_id": file_id, "journalist_id": journalist_id} - sql = """ - INSERT INTO seen_files (file_id, journalist_id) - VALUES (:file_id, :journalist_id) - """ - session.execute(text(sql), params) - - -def mark_message_as_seen(session: Session, message_id: int, journalist_id: int) -> None: - params = {"message_id": message_id, "journalist_id": journalist_id} - sql = """ - INSERT INTO seen_messages (message_id, journalist_id) - VALUES (:message_id, :journalist_id) - """ - session.execute(text(sql), params) - - -def mark_reply_as_seen(session: Session, reply_id: int, journalist_id: int): - params = {"reply_id": reply_id, "journalist_id": journalist_id} - sql = """ - INSERT INTO seen_replies (reply_id, journalist_id) - VALUES (:reply_id, :journalist_id) - """ - session.execute(text(sql), params) - - -def add_draft_reply(session: Session, journalist_id: int, source_id: int) -> None: - reply_send_statuses = session.query(ReplySendStatus).all() - reply_send_status_ids = [reply_send_status.id for reply_send_status in reply_send_statuses] - - content = random_chars(1000) - - source = session.query(Source).filter_by(id=source_id).one() - - file_counter = len(source.collection) + 1 - - params = { - "uuid": str(uuid4()), - "journalist_id": journalist_id, - "source_id": source_id, - "file_counter": file_counter, - "content": content, - "send_status_id": random.choice(reply_send_status_ids), - "timestamp": random_datetime(), - } - - sql = """ - INSERT INTO draftreplies - ( - uuid, - journalist_id, - source_id, - file_counter, - content, - send_status_id, - timestamp - ) - VALUES - ( - :uuid, - :journalist_id, - :source_id, - :file_counter, - :content, - :send_status_id, - :timestamp - ) - """ - session.execute(text(sql), params) From b3cecdc85f9badc6924714053ef7851de9a9b8d1 Mon Sep 17 00:00:00 2001 From: Cory Francis Myers Date: Mon, 13 Jun 2022 18:49:31 -0700 Subject: [PATCH 05/10] docs: Alembic migrations and tests must be self-contained --- .github/pull_request_template.md | 4 +++- README.md | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e9c845e43..505661ad6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -21,7 +21,9 @@ If these changes add or remove files other than client code, the AppArmor profil If these changes modify the database schema, you should include a database migration. Please check as applicable: - - [ ] I have written a migration and upgraded a test database based on `main` and confirmed that the migration applies cleanly + - [ ] I have written a migration and upgraded a test database based on `main` and confirmed that the migration is [self-contained] and applies cleanly - [ ] I have written a migration but have not upgraded a test database based on `main` and would like the reviewer to do so - [ ] I need help writing a database migration - [ ] No database schema changes are needed + +[self-contained]: https://github.com/freedomofpress/securedrop-client#generating-and-running-database-migrations diff --git a/README.md b/README.md index daf187682..948a7814b 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,8 @@ alembic upgrade head alembic revision --autogenerate -m "describe your revision here" ``` +**NOTE.** Schema migrations, data migrations, and tests [MUST] be self-contained. That is, their `upgrade()` and `downgrade()` methods and their tests [MUST NOT] rely, directly or indirectly, on other project code, such as `db.py`'s SQLAlchemy models or other helper classes and functions defined outside of the migration under test, because these utilities may change in Git over time. (The scaffolding of the `test_alembic.py` test suite [MAY] rely on such utilities, because it is versioned at the Git level, not the Alembic level.) See [#1500](https://github.com/freedomofpress/securedrop-client/issues/1500) for an example of why this guideline applies. + ## AppArmor support An AppArmor profile is available for mandatory access control. When installing securedrop-client from a .deb package, the AppArmor profile will automatically be copied and enforced. Below are instructions to use the profile in non-production scenarios. @@ -406,3 +408,8 @@ Then you can use [`pdb` commands](https://docs.python.org/3/library/pdb.html#deb Logs can be found in the `{sdc-home}/logs`. If you are debugging a version of this application installed from a deb package in Qubes, you can debug issues by looking at the log file in `~/.securedrop_client/logs/client.log`. You can also add additional log lines in the running code in `/opt/venvs/securedrop-client/lib/python3.7/site-packages/securedrop_client/`. + + +[MAY]: https://datatracker.ietf.org/doc/html/rfc2119#section-5 +[MUST]: https://datatracker.ietf.org/doc/html/rfc2119#section-1 +[MUST NOT]: https://datatracker.ietf.org/doc/html/rfc2119#section-2 From 02eef05fc4d64564d4284d5f43436bb4b174c31a Mon Sep 17 00:00:00 2001 From: Cory Francis Myers Date: Mon, 27 Jun 2022 12:20:59 -0700 Subject: [PATCH 06/10] refactor: data-migration tests now use their own session factory The test scaffolding in tests.test_alembic shouldn't be coupled to the implementation under test in securedrop_client.db. --- tests/test_alembic.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/test_alembic.py b/tests/test_alembic.py index b165a5c4c..70d734fb7 100644 --- a/tests/test_alembic.py +++ b/tests/test_alembic.py @@ -6,11 +6,12 @@ from os import path import pytest -from sqlalchemy import text +from sqlalchemy import create_engine, text +from sqlalchemy.orm import scoped_session, sessionmaker from alembic.config import Config as AlembicConfig from alembic.script import ScriptDirectory -from securedrop_client.db import Base, convention, make_session_maker +from securedrop_client.db import Base, convention from . import conftest @@ -25,6 +26,17 @@ WHITESPACE_REGEX = re.compile(r"\s+") +def make_session_maker(home: str) -> scoped_session: + """ + Duplicate securedrop_client.db.make_session_maker so that data migrations are decoupled + from that implementation. + """ + db_path = os.path.join(home, "svs.sqlite") + engine = create_engine("sqlite:///{}".format(db_path)) + maker = sessionmaker(bind=engine) + return scoped_session(maker) + + def list_migrations(cfg_path, head): cfg = AlembicConfig(cfg_path) script = ScriptDirectory.from_config(cfg) @@ -149,7 +161,8 @@ def test_alembic_migration_upgrade_with_data(alembic_config, config, migration, upgrade(alembic_config, migrations[-2]) mod_name = "tests.migrations.test_{}".format(migration) mod = __import__(mod_name, fromlist=["UpgradeTester"]) - upgrade_tester = mod.UpgradeTester(homedir) + session = make_session_maker(homedir) + upgrade_tester = mod.UpgradeTester(homedir, session) upgrade_tester.load_data() upgrade(alembic_config, migration) upgrade_tester.check_upgrade() @@ -177,7 +190,8 @@ def test_alembic_migration_downgrade_with_data(alembic_config, config, migration upgrade(alembic_config, migration) mod_name = "tests.migrations.test_{}".format(migration) mod = __import__(mod_name, fromlist=["DowngradeTester"]) - downgrade_tester = mod.DowngradeTester(homedir) + session = make_session_maker(homedir) + downgrade_tester = mod.DowngradeTester(homedir, session) downgrade_tester.load_data() downgrade(alembic_config, "-1") downgrade_tester.check_downgrade() From 0add5e3c8ebdd53f2094c8379903d36778860bc0 Mon Sep 17 00:00:00 2001 From: Cory Francis Myers Date: Mon, 27 Jun 2022 12:25:19 -0700 Subject: [PATCH 07/10] test: Alembic base migration d7c8af95bc8e populates enum tables --- tests/migrations/test_d7c8af95bc8e.py | 47 +++++++++++++++++++++++++++ tests/test_alembic.py | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 tests/migrations/test_d7c8af95bc8e.py diff --git a/tests/migrations/test_d7c8af95bc8e.py b/tests/migrations/test_d7c8af95bc8e.py new file mode 100644 index 000000000..59e99b768 --- /dev/null +++ b/tests/migrations/test_d7c8af95bc8e.py @@ -0,0 +1,47 @@ +import os +import subprocess + +import pytest +from sqlalchemy import text +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import scoped_session + +ENUMS = { + "downloaderrors": ["CHECKSUM_ERROR", "DECRYPTION_ERROR"], + "replysendstatuses": ["PENDING", "FAILED"], +} + + +class UpgradeTester: + """Enum tables exist and contains the expected values.""" + + def __init__(self, homedir: str, session: scoped_session) -> None: + subprocess.check_call(["sqlite3", os.path.join(homedir, "svs.sqlite"), ".databases"]) + self.session = session + + def load_data(self): + pass + + def check_upgrade(self): + for table, values in ENUMS.items(): + for value in values: + result = self.session.execute( + text(f"SELECT * FROM {table} WHERE name = '{value}'") + ).fetchone() + assert result is not None + + +class DowngradeTester: + """Enum tables do not exist.""" + + def __init__(self, homedir: str, session: scoped_session) -> None: + subprocess.check_call(["sqlite3", os.path.join(homedir, "svs.sqlite"), ".databases"]) + self.session = session + + def load_data(self): + pass + + def check_downgrade(self): + for table in ENUMS: + with pytest.raises(OperationalError): + self.session.execute(text(f"SELECT name FROM {table}")) diff --git a/tests/test_alembic.py b/tests/test_alembic.py index 70d734fb7..394cca0c4 100644 --- a/tests/test_alembic.py +++ b/tests/test_alembic.py @@ -21,7 +21,7 @@ x.split(".")[0].split("_")[0] for x in os.listdir(MIGRATION_PATH) if x.endswith(".py") ] -DATA_MIGRATIONS = [] +DATA_MIGRATIONS = ["d7c8af95bc8e"] WHITESPACE_REGEX = re.compile(r"\s+") From 3bc663f7bd37bcada3d56ec6496abc9b992e1e36 Mon Sep 17 00:00:00 2001 From: Cory Francis Myers Date: Mon, 27 Jun 2022 15:35:56 -0700 Subject: [PATCH 08/10] test: support testing data migration in base migration --- tests/test_alembic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_alembic.py b/tests/test_alembic.py index 394cca0c4..487a324f3 100644 --- a/tests/test_alembic.py +++ b/tests/test_alembic.py @@ -152,13 +152,13 @@ def test_alembic_migration_upgrade(alembic_config, config, migration): @pytest.mark.parametrize("migration", DATA_MIGRATIONS) def test_alembic_migration_upgrade_with_data(alembic_config, config, migration, homedir): """ - Upgrade to one migration before the target migration, load data, then upgrade in order to test - that the upgrade is successful when there is data. + Upgrade to one migration before the target migration (if there is one), + load data, then upgrade in order to test that the upgrade is successful + when there is data. """ migrations = list_migrations(alembic_config, migration) - if len(migrations) == 1: - return - upgrade(alembic_config, migrations[-2]) + if len(migrations) > 1: + upgrade(alembic_config, migrations[-2]) mod_name = "tests.migrations.test_{}".format(migration) mod = __import__(mod_name, fromlist=["UpgradeTester"]) session = make_session_maker(homedir) From bc9c8590b8df00e7695332591b0ed4d2b7b6f5c0 Mon Sep 17 00:00:00 2001 From: Cory Francis Myers Date: Mon, 27 Jun 2022 15:51:00 -0700 Subject: [PATCH 09/10] fix: restore enum values --- alembic/versions/d7c8af95bc8e_initial.py | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/alembic/versions/d7c8af95bc8e_initial.py b/alembic/versions/d7c8af95bc8e_initial.py index f41334cf0..5eba89284 100644 --- a/alembic/versions/d7c8af95bc8e_initial.py +++ b/alembic/versions/d7c8af95bc8e_initial.py @@ -277,6 +277,31 @@ def upgrade(): ) # ### end Alembic commands ### + # Set enum values: + conn = op.get_bind() + + # downloaderrors (from original Alembic version 7f682532afa2) + conn.execute( + """ + INSERT INTO downloaderrors + ('name') + VALUES + ('CHECKSUM_ERROR'), + ('DECRYPTION_ERROR'); + """ + ) + + # replysendstatuses (from original Alembic version 86b01b6290da) + conn.execute( + """ + INSERT INTO replysendstatuses + ('name') + VALUES + ('PENDING'), + ('FAILED'); + """ + ) + def downgrade(): # ### commands auto generated by Alembic - please adjust! ### From 658a04cf97c5868689a659717d369615e27e31ab Mon Sep 17 00:00:00 2001 From: Cory Francis Myers Date: Thu, 30 Jun 2022 13:00:00 -0700 Subject: [PATCH 10/10] style: reorder table and column operations to minimize SQL diff The SQL statements generated by "sqlite3 svs.sqlite .dump" replicate the database's tables and columns in the order in which they were originally added. For the current database schema, that's a hybrid of the order in which they're defined in securedrop_client.db and subsequent additions in migrations; in the new database schema, that's strictly their ordering in securedrop_client.db. The latter artificially inflates the diff of comparing "sqlite3 svs.sqlite .dump" for the old and new schemas. For ease of review, here we attempt to replicate the current SQL statements as closely as possible, for as small a diff as possible. --- alembic/versions/d7c8af95bc8e_initial.py | 72 ++++++++++++------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/alembic/versions/d7c8af95bc8e_initial.py b/alembic/versions/d7c8af95bc8e_initial.py index 5eba89284..3b39bcc20 100644 --- a/alembic/versions/d7c8af95bc8e_initial.py +++ b/alembic/versions/d7c8af95bc8e_initial.py @@ -18,30 +18,6 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "deletedconversation", - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedconversation")), - ) - op.create_table( - "deletedsource", - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedsource")), - ) - op.create_table( - "downloaderrors", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("name", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_downloaderrors")), - sa.UniqueConstraint("name", name=op.f("uq_downloaderrors_name")), - ) - op.create_table( - "replysendstatuses", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("name", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_replysendstatuses")), - sa.UniqueConstraint("name", name=op.f("uq_replysendstatuses_name")), - ) op.create_table( "sources", sa.Column("id", sa.Integer(), nullable=False), @@ -71,6 +47,13 @@ def upgrade(): sa.PrimaryKeyConstraint("id", name=op.f("pk_users")), sa.UniqueConstraint("uuid", name=op.f("uq_users_uuid")), ) + op.create_table( + "replysendstatuses", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_replysendstatuses")), + sa.UniqueConstraint("name", name=op.f("uq_replysendstatuses_name")), + ) op.create_table( "draftreplies", sa.Column("id", sa.Integer(), nullable=False), @@ -81,6 +64,10 @@ def upgrade(): sa.Column("file_counter", sa.Integer(), nullable=False), sa.Column("content", sa.Text(), nullable=True), sa.Column("send_status_id", sa.Integer(), nullable=True), + sa.UniqueConstraint("uuid", name=op.f("uq_draftreplies_uuid")), + sa.ForeignKeyConstraint( + ["source_id"], ["sources.id"], name=op.f("fk_draftreplies_source_id_sources") + ), sa.ForeignKeyConstraint( ["journalist_id"], ["users.id"], name=op.f("fk_draftreplies_journalist_id_users") ), @@ -89,11 +76,14 @@ def upgrade(): ["replysendstatuses.id"], name=op.f("fk_draftreplies_send_status_id_replysendstatuses"), ), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_draftreplies_source_id_sources") - ), sa.PrimaryKeyConstraint("id", name=op.f("pk_draftreplies")), - sa.UniqueConstraint("uuid", name=op.f("uq_draftreplies_uuid")), + ) + op.create_table( + "downloaderrors", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_downloaderrors")), + sa.UniqueConstraint("name", name=op.f("uq_downloaderrors_name")), ) op.create_table( "files", @@ -124,6 +114,9 @@ def upgrade(): ), sa.Column("source_id", sa.Integer(), nullable=False), sa.Column("last_updated", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_files")), + sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), + sa.UniqueConstraint("uuid", name=op.f("uq_files_uuid")), sa.ForeignKeyConstraint( ["download_error_id"], ["downloaderrors.id"], @@ -132,9 +125,6 @@ def upgrade(): sa.ForeignKeyConstraint( ["source_id"], ["sources.id"], name=op.f("fk_files_source_id_sources") ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_files")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_files_uuid")), ) op.create_table( "messages", @@ -174,6 +164,9 @@ def upgrade(): ), sa.Column("source_id", sa.Integer(), nullable=False), sa.Column("last_updated", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), + sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), + sa.UniqueConstraint("uuid", name=op.f("uq_messages_uuid")), sa.ForeignKeyConstraint( ["download_error_id"], ["downloaderrors.id"], @@ -182,20 +175,15 @@ def upgrade(): sa.ForeignKeyConstraint( ["source_id"], ["sources.id"], name=op.f("fk_messages_source_id_sources") ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_messages_uuid")), ) op.create_table( "replies", sa.Column("id", sa.Integer(), nullable=False), sa.Column("uuid", sa.String(length=36), nullable=False), sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), sa.Column("filename", sa.String(length=255), nullable=False), sa.Column("file_counter", sa.Integer(), nullable=False), sa.Column("size", sa.Integer(), nullable=True), - sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), sa.Column( "content", sa.Text(), @@ -214,7 +202,9 @@ def upgrade(): ), nullable=True, ), + sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), sa.Column("download_error_id", sa.Integer(), nullable=True), + sa.Column("journalist_id", sa.Integer(), nullable=True), sa.Column("last_updated", sa.DateTime(), nullable=False), sa.ForeignKeyConstraint( ["download_error_id"], @@ -275,6 +265,16 @@ def upgrade(): sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_replies")), sa.UniqueConstraint("reply_id", "journalist_id", name=op.f("uq_seen_replies_reply_id")), ) + op.create_table( + "deletedconversation", + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedconversation")), + ) + op.create_table( + "deletedsource", + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedsource")), + ) # ### end Alembic commands ### # Set enum values: