diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e9c845e43..505661ad6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -21,7 +21,9 @@ If these changes add or remove files other than client code, the AppArmor profil If these changes modify the database schema, you should include a database migration. Please check as applicable: - - [ ] I have written a migration and upgraded a test database based on `main` and confirmed that the migration applies cleanly + - [ ] I have written a migration and upgraded a test database based on `main` and confirmed that the migration is [self-contained] and applies cleanly - [ ] I have written a migration but have not upgraded a test database based on `main` and would like the reviewer to do so - [ ] I need help writing a database migration - [ ] No database schema changes are needed + +[self-contained]: https://github.com/freedomofpress/securedrop-client#generating-and-running-database-migrations diff --git a/README.md b/README.md index daf187682..948a7814b 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,8 @@ alembic upgrade head alembic revision --autogenerate -m "describe your revision here" ``` +**NOTE.** Schema migrations, data migrations, and tests [MUST] be self-contained. That is, their `upgrade()` and `downgrade()` methods and their tests [MUST NOT] rely, directly or indirectly, on other project code, such as `db.py`'s SQLAlchemy models or other helper classes and functions defined outside of the migration under test, because these utilities may change in Git over time. (The scaffolding of the `test_alembic.py` test suite [MAY] rely on such utilities, because it is versioned at the Git level, not the Alembic level.) See [#1500](https://github.com/freedomofpress/securedrop-client/issues/1500) for an example of why this guideline applies. + ## AppArmor support An AppArmor profile is available for mandatory access control. When installing securedrop-client from a .deb package, the AppArmor profile will automatically be copied and enforced. Below are instructions to use the profile in non-production scenarios. @@ -406,3 +408,8 @@ Then you can use [`pdb` commands](https://docs.python.org/3/library/pdb.html#deb Logs can be found in the `{sdc-home}/logs`. If you are debugging a version of this application installed from a deb package in Qubes, you can debug issues by looking at the log file in `~/.securedrop_client/logs/client.log`. You can also add additional log lines in the running code in `/opt/venvs/securedrop-client/lib/python3.7/site-packages/securedrop_client/`. + + +[MAY]: https://datatracker.ietf.org/doc/html/rfc2119#section-5 +[MUST]: https://datatracker.ietf.org/doc/html/rfc2119#section-1 +[MUST NOT]: https://datatracker.ietf.org/doc/html/rfc2119#section-2 diff --git a/alembic/versions/2f363b3d680e_init.py b/alembic/versions/2f363b3d680e_init.py deleted file mode 100644 index 8e1b2cede..000000000 --- a/alembic/versions/2f363b3d680e_init.py +++ /dev/null @@ -1,161 +0,0 @@ -"""init - -Revision ID: 2f363b3d680e -Revises: -Create Date: 2019-02-08 12:07:47.062397 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "2f363b3d680e" -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - "sources", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("journalist_designation", sa.String(length=255), nullable=False), - sa.Column("document_count", sa.Integer(), server_default=sa.text("0"), nullable=False), - sa.Column( - "is_flagged", sa.Boolean(name="is_flagged"), server_default=sa.text("0"), nullable=True - ), - sa.Column("public_key", sa.Text(), nullable=True), - sa.Column("fingerprint", sa.String(length=64), nullable=True), - sa.Column("interaction_count", sa.Integer(), server_default=sa.text("0"), nullable=False), - sa.Column( - "is_starred", sa.Boolean(name="is_starred"), server_default=sa.text("0"), nullable=True - ), - sa.Column("last_updated", sa.DateTime(), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_sources")), - sa.UniqueConstraint("uuid", name=op.f("uq_sources_uuid")), - ) - - op.create_table( - "users", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("username", sa.String(length=255), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_users")), - sa.UniqueConstraint("uuid", name=op.f("uq_users_uuid")), - ) - - op.create_table( - "files", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=False), - sa.Column("download_url", sa.String(length=255), nullable=False), - sa.Column( - "is_downloaded", - sa.Boolean(name="is_downloaded"), - server_default=sa.text("0"), - nullable=False, - ), - sa.Column( - "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False - ), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_files_source_id_sources") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_files")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_files_uuid")), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="files_compare_is_downloaded_vs_is_decrypted", - ), - ) - - op.create_table( - "messages", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=False), - sa.Column("download_url", sa.String(length=255), nullable=False), - sa.Column( - "is_downloaded", - sa.Boolean(name="is_downloaded"), - server_default=sa.text("0"), - nullable=False, - ), - sa.Column( - "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False - ), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_messages_source_id_sources") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_messages_uuid")), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name=op.f("ck_message_compare_download_vs_content"), - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="messages_compare_is_downloaded_vs_is_decrypted", - ), - sa.CheckConstraint( - "CASE WHEN is_decrypted = 0 THEN content IS NULL ELSE 1 END", - name="messages_compare_is_decrypted_vs_content", - ), - ) - - op.create_table( - "replies", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=True), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_replies_journalist_id_users") - ), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_replies_source_id_sources") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_replies")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_replies_uuid")), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name="replies_compare_download_vs_content", - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="replies_compare_is_downloaded_vs_is_decrypted", - ), - sa.CheckConstraint( - "CASE WHEN is_decrypted = 0 THEN content IS NULL ELSE 1 END", - name="replies_compare_is_decrypted_vs_content", - ), - ) - - -def downgrade(): - op.drop_table("replies") - op.drop_table("messages") - op.drop_table("files") - op.drop_table("users") - op.drop_table("sources") diff --git a/alembic/versions/36a79ffcfbfb_add_first_name_last_name_fullname_.py b/alembic/versions/36a79ffcfbfb_add_first_name_last_name_fullname_.py deleted file mode 100644 index 302153fba..000000000 --- a/alembic/versions/36a79ffcfbfb_add_first_name_last_name_fullname_.py +++ /dev/null @@ -1,27 +0,0 @@ -"""add first_name, last_name, fullname, initials - -Revision ID: 36a79ffcfbfb -Revises: fecf1191b6f0 -Create Date: 2019-06-28 15:21:50.893256 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "36a79ffcfbfb" -down_revision = "bafdcae12f97" -branch_labels = None -depends_on = None - - -def upgrade(): - op.add_column("users", sa.Column("firstname", sa.String(length=64), nullable=True)) - op.add_column("users", sa.Column("lastname", sa.String(length=64), nullable=True)) - - -def downgrade(): - with op.batch_alter_table("users", schema=None) as batch_op: - batch_op.drop_column("lastname") - batch_op.drop_column("firstname") diff --git a/alembic/versions/7f682532afa2_add_download_error.py b/alembic/versions/7f682532afa2_add_download_error.py deleted file mode 100644 index b0982f846..000000000 --- a/alembic/versions/7f682532afa2_add_download_error.py +++ /dev/null @@ -1,302 +0,0 @@ -"""add download error - -Revision ID: 7f682532afa2 -Revises: fb657f2ee8a7 -Create Date: 2020-04-15 13:44:21.434312 - -""" -import sqlalchemy as sa - -from alembic import op -from securedrop_client import db - -# revision identifiers, used by Alembic. -revision = "7f682532afa2" -down_revision = "fb657f2ee8a7" -branch_labels = None -depends_on = None - - -CREATE_TABLE_FILES_NEW = """ - CREATE TABLE files ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER NOT NULL, - download_url VARCHAR(255) NOT NULL, - is_downloaded BOOLEAN DEFAULT 0 NOT NULL, - is_decrypted BOOLEAN CONSTRAINT files_compare_is_downloaded_vs_is_decrypted CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - download_error_id INTEGER, - is_read BOOLEAN DEFAULT 0 NOT NULL, - source_id INTEGER NOT NULL, - last_updated DATETIME NOT NULL, - CONSTRAINT pk_files PRIMARY KEY (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_files_uuid UNIQUE (uuid), - CONSTRAINT ck_files_is_downloaded CHECK (is_downloaded IN (0, 1)), - CONSTRAINT ck_files_is_decrypted CHECK (is_decrypted IN (0, 1)), - CONSTRAINT fk_files_download_error_id_downloaderrors FOREIGN KEY(download_error_id) REFERENCES downloaderrors (id), - CONSTRAINT ck_files_is_read CHECK (is_read IN (0, 1)), - CONSTRAINT fk_files_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id) -); -""" - -CREATE_TABLE_FILES_OLD = """ - CREATE TABLE files ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER NOT NULL, - download_url VARCHAR(255) NOT NULL, - is_downloaded BOOLEAN DEFAULT 0 NOT NULL, - is_read BOOLEAN DEFAULT 0 NOT NULL, - is_decrypted BOOLEAN, - source_id INTEGER NOT NULL, - CONSTRAINT pk_files PRIMARY KEY (id), - CONSTRAINT fk_files_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_files_uuid UNIQUE (uuid), - CONSTRAINT files_compare_is_downloaded_vs_is_decrypted - CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - CONSTRAINT ck_files_is_downloaded CHECK (is_downloaded IN (0, 1)), - CONSTRAINT ck_files_is_read CHECK (is_read IN (0, 1)), - CONSTRAINT ck_files_is_decrypted CHECK (is_decrypted IN (0, 1)) -); -""" - - -CREATE_TABLE_MESSAGES_NEW = """ - CREATE TABLE messages ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER NOT NULL, - download_url VARCHAR(255) NOT NULL, - is_downloaded BOOLEAN DEFAULT 0 NOT NULL, - is_decrypted BOOLEAN CONSTRAINT messages_compare_is_downloaded_vs_is_decrypted CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - download_error_id INTEGER, - is_read BOOLEAN DEFAULT 0 NOT NULL, - content TEXT CONSTRAINT ck_message_compare_download_vs_content CHECK (CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END), - source_id INTEGER NOT NULL, - last_updated DATETIME NOT NULL, - CONSTRAINT pk_messages PRIMARY KEY (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_messages_uuid UNIQUE (uuid), - CONSTRAINT ck_messages_is_downloaded CHECK (is_downloaded IN (0, 1)), - CONSTRAINT ck_messages_is_decrypted CHECK (is_decrypted IN (0, 1)), - CONSTRAINT fk_messages_download_error_id_downloaderrors FOREIGN KEY(download_error_id) REFERENCES downloaderrors (id), - CONSTRAINT ck_messages_is_read CHECK (is_read IN (0, 1)), - CONSTRAINT fk_messages_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id) - ); -""" - -CREATE_TABLE_MESSAGES_OLD = """ - CREATE TABLE messages ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - source_id INTEGER NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER NOT NULL, - content TEXT, - is_decrypted BOOLEAN, - is_downloaded BOOLEAN DEFAULT 0 NOT NULL, - is_read BOOLEAN DEFAULT 0 NOT NULL, - download_url VARCHAR(255) NOT NULL, - CONSTRAINT pk_messages PRIMARY KEY (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_messages_uuid UNIQUE (uuid), - CONSTRAINT fk_messages_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id), - CONSTRAINT ck_message_compare_download_vs_content - CHECK (CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END), - CONSTRAINT messages_compare_is_downloaded_vs_is_decrypted - CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - CONSTRAINT ck_messages_is_decrypted CHECK (is_decrypted IN (0, 1)), - CONSTRAINT ck_messages_is_downloaded CHECK (is_downloaded IN (0, 1)), - CONSTRAINT ck_messages_is_read CHECK (is_read IN (0, 1)) - ); -""" - - -CREATE_TABLE_REPLIES_NEW = """ - CREATE TABLE replies ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - source_id INTEGER NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER, - content TEXT, - is_decrypted BOOLEAN, - is_downloaded BOOLEAN, - download_error_id INTEGER, - journalist_id INTEGER, - last_updated DATETIME NOT NULL, - CONSTRAINT pk_replies PRIMARY KEY (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_replies_uuid UNIQUE (uuid), - CONSTRAINT fk_replies_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id), - CONSTRAINT fk_replies_download_error_id_downloaderrors - FOREIGN KEY(download_error_id) REFERENCES downloaderrors (id), - CONSTRAINT fk_replies_journalist_id_users FOREIGN KEY(journalist_id) REFERENCES users (id), - CONSTRAINT replies_compare_download_vs_content - CHECK (CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END), - CONSTRAINT replies_compare_is_downloaded_vs_is_decrypted - CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - CONSTRAINT ck_replies_is_decrypted CHECK (is_decrypted IN (0, 1)), - CONSTRAINT ck_replies_is_downloaded CHECK (is_downloaded IN (0, 1)) - ); -""" - -CREATE_TABLE_REPLIES_OLD = """ - CREATE TABLE replies ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - source_id INTEGER NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER, - content TEXT, - is_decrypted BOOLEAN, - is_downloaded BOOLEAN, - journalist_id INTEGER, - CONSTRAINT pk_replies PRIMARY KEY (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_replies_uuid UNIQUE (uuid), - CONSTRAINT fk_replies_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id), - CONSTRAINT fk_replies_journalist_id_users FOREIGN KEY(journalist_id) REFERENCES users (id), - CONSTRAINT replies_compare_download_vs_content - CHECK (CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END), - CONSTRAINT replies_compare_is_downloaded_vs_is_decrypted - CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - CONSTRAINT ck_replies_is_decrypted CHECK (is_decrypted IN (0, 1)), - CONSTRAINT ck_replies_is_downloaded CHECK (is_downloaded IN (0, 1)) - ); -""" - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "downloaderrors", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("name", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_downloaderrors")), - sa.UniqueConstraint("name", name=op.f("uq_downloaderrors_name")), - ) - - conn = op.get_bind() - for name, member in db.DownloadErrorCodes.__members__.items(): - conn.execute("""INSERT INTO downloaderrors (name) VALUES (:name);""", name) - - op.rename_table("files", "files_tmp") - op.rename_table("messages", "messages_tmp") - op.rename_table("replies", "replies_tmp") - - conn.execute(CREATE_TABLE_FILES_NEW) - conn.execute(CREATE_TABLE_MESSAGES_NEW) - conn.execute(CREATE_TABLE_REPLIES_NEW) - - conn.execute( - """ - INSERT INTO files - ( - id, uuid, filename, file_counter, size, download_url, - is_downloaded, is_read, is_decrypted, download_error_id, source_id, - last_updated - ) - SELECT id, uuid, filename, file_counter, size, download_url, - is_downloaded, is_read, is_decrypted, NULL, source_id, CURRENT_TIMESTAMP - FROM files_tmp - """ - ) - - conn.execute( - """ - INSERT INTO messages - ( - id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, is_read, download_error_id, download_url, last_updated - ) - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, is_read, NULL, download_url, CURRENT_TIMESTAMP - FROM messages_tmp - """ - ) - - conn.execute( - """ - INSERT INTO replies - ( - id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, download_error_id, journalist_id, last_updated - ) - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, NULL, journalist_id, CURRENT_TIMESTAMP - FROM replies_tmp - """ - ) - - # Delete the old tables. - op.drop_table("files_tmp") - op.drop_table("messages_tmp") - op.drop_table("replies_tmp") - - # ### end Alembic commands ### - - -def downgrade(): - - conn = op.get_bind() - - op.rename_table("files", "files_tmp") - op.rename_table("messages", "messages_tmp") - op.rename_table("replies", "replies_tmp") - - conn.execute(CREATE_TABLE_FILES_OLD) - conn.execute(CREATE_TABLE_MESSAGES_OLD) - conn.execute(CREATE_TABLE_REPLIES_OLD) - - conn.execute( - """ - INSERT INTO files - (id, uuid, filename, file_counter, size, download_url, - is_downloaded, is_read, is_decrypted, source_id) - SELECT id, uuid, filename, file_counter, size, download_url, - is_downloaded, is_read, is_decrypted, source_id - FROM files_tmp - """ - ) - conn.execute( - """ - INSERT INTO messages - (id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, is_read, download_url) - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, is_read, download_url - FROM messages_tmp - """ - ) - conn.execute( - """ - INSERT INTO replies - (id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, journalist_id) - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, journalist_id - FROM replies_tmp - """ - ) - - # Delete the old tables. - op.drop_table("files_tmp") - op.drop_table("messages_tmp") - op.drop_table("replies_tmp") - - # Drop downloaderrors - op.drop_table("downloaderrors") - - # ### end Alembic commands ### diff --git a/alembic/versions/86b01b6290da_add_reply_draft.py b/alembic/versions/86b01b6290da_add_reply_draft.py deleted file mode 100644 index 77d5f4ba7..000000000 --- a/alembic/versions/86b01b6290da_add_reply_draft.py +++ /dev/null @@ -1,68 +0,0 @@ -"""add reply draft - -Revision ID: 86b01b6290da -Revises: 36a79ffcfbfb -Create Date: 2019-10-17 09:45:07.643076 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "86b01b6290da" -down_revision = "36a79ffcfbfb" -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - "replysendstatuses", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("name", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_replysendstatuses")), - sa.UniqueConstraint("name", name=op.f("uq_replysendstatuses_name")), - ) - - # Set the initial in-progress send statuses: PENDING, FAILED - conn = op.get_bind() - conn.execute( - """ - INSERT INTO replysendstatuses - ('name') - VALUES - ('PENDING'), - ('FAILED'); - """ - ) - - op.create_table( - "draftreplies", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("timestamp", sa.DateTime(), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("send_status_id", sa.Integer(), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_draftreplies")), - sa.UniqueConstraint("uuid", name=op.f("uq_draftreplies_uuid")), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_draftreplies_source_id_sources") - ), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_draftreplies_journalist_id_users") - ), - sa.ForeignKeyConstraint( - ["send_status_id"], - ["replysendstatuses.id"], - op.f("fk_draftreplies_send_status_id_replysendstatuses"), - ), - ) - - -def downgrade(): - op.drop_table("draftreplies") - op.drop_table("replysendstatuses") diff --git a/alembic/versions/9ba8d7524871_add_deletedsource_table.py b/alembic/versions/9ba8d7524871_add_deletedsource_table.py deleted file mode 100644 index cf2aebd92..000000000 --- a/alembic/versions/9ba8d7524871_add_deletedsource_table.py +++ /dev/null @@ -1,32 +0,0 @@ -"""add deletedsource table - -Revision ID: 9ba8d7524871 -Revises: eff1387cfd0b -Create Date: 2022-03-15 12:25:59.145300 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "9ba8d7524871" -down_revision = "eff1387cfd0b" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### Add DeletedSource table ### - op.create_table( - "deletedsource", - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedsource")), - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### Remove DeletedSource table ### - op.drop_table("deletedsource") - # ### end Alembic commands ### diff --git a/alembic/versions/a4bf1f58ce69_fix_journalist_association_in_replies.py b/alembic/versions/a4bf1f58ce69_fix_journalist_association_in_replies.py deleted file mode 100644 index 35bbaecb9..000000000 --- a/alembic/versions/a4bf1f58ce69_fix_journalist_association_in_replies.py +++ /dev/null @@ -1,85 +0,0 @@ -"""fix journalist association in replies table - -Revision ID: a4bf1f58ce69 -Revises: 7f682532afa2 -Create Date: 2020-10-20 13:49:53.035383 - -""" -from alembic import op - -# revision identifiers, used by Alembic. -revision = "a4bf1f58ce69" -down_revision = "7f682532afa2" -branch_labels = None -depends_on = None - - -def upgrade(): - """ - Fix reply association with journalist by updating journalist uuid to journalist id in the - journalist_id column for the replies and draftreplies tables. - """ - conn = op.get_bind() - cursor = conn.execute( - """ - SELECT journalist_id - FROM replies, users - WHERE journalist_id=users.uuid; - """ - ) - - replies_with_incorrect_associations = cursor.fetchall() - if replies_with_incorrect_associations: - conn.execute( - """ - UPDATE replies - SET journalist_id= - ( - SELECT users.id - FROM users - WHERE journalist_id=users.uuid - ) - WHERE exists - ( - SELECT users.id - FROM users - WHERE journalist_id=users.uuid - ); - """ - ) - - cursor = conn.execute( - """ - SELECT journalist_id - FROM draftreplies, users - WHERE journalist_id=users.uuid; - """ - ) - - draftreplies_with_incorrect_associations = cursor.fetchall() - if draftreplies_with_incorrect_associations: - conn.execute( - """ - UPDATE draftreplies - SET journalist_id= - ( - SELECT users.id - FROM users - WHERE journalist_id=users.uuid - ) - WHERE exists - ( - SELECT users.id - FROM users - WHERE journalist_id=users.uuid - ); - """ - ) - - -def downgrade(): - """ - We do not want to undo this bug fix, because nothing will break if we downgrade to an earlier - version of the client. - """ - pass diff --git a/alembic/versions/bafdcae12f97_.py b/alembic/versions/bafdcae12f97_.py deleted file mode 100644 index 9e9408a78..000000000 --- a/alembic/versions/bafdcae12f97_.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Add files.original_filename - -Revision ID: bafdcae12f97 -Revises: fecf1191b6f0 -Create Date: 2019-06-24 13:45:47.239212 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "bafdcae12f97" -down_revision = "fecf1191b6f0" -branch_labels = None -depends_on = None - - -def upgrade(): - op.add_column( - "files", - sa.Column("original_filename", sa.String(length=255), nullable=False, server_default=""), - ) - - conn = op.get_bind() - conn.execute("""UPDATE files SET original_filename = replace(filename, '.gz.gpg', '');""") - - -def downgrade(): - with op.batch_alter_table("files", schema=None) as batch_op: - batch_op.drop_column("original_filename") diff --git a/alembic/versions/bd57477f19a2_add_seen_tables.py b/alembic/versions/bd57477f19a2_add_seen_tables.py deleted file mode 100644 index 05809069f..000000000 --- a/alembic/versions/bd57477f19a2_add_seen_tables.py +++ /dev/null @@ -1,78 +0,0 @@ -"""add seen tables - -Revision ID: bd57477f19a2 -Revises: a4bf1f58ce69 -Create Date: 2020-10-20 22:43:46.743035 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "bd57477f19a2" -down_revision = "a4bf1f58ce69" -branch_labels = None -depends_on = None - - -def upgrade(): - """ - Create seen tables for files, messages, and replies. - - Once freedomofpress/securedrop#5503 is fixed, we can expect that journalist_id will never be - NULL and then migrate these tables so that both of the foreign keys make up the primary key. - We will also need to migrate any existing NULL journalist IDs in these tables (as well as the - 'replies' table) to a non-NULL id, most likely to the ID of a special global "deleted" User - only to be used for historical data. - """ - op.create_table( - "seen_files", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("file_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.ForeignKeyConstraint( - ["file_id"], ["files.id"], name=op.f("fk_seen_files_file_id_files") - ), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_seen_files_journalist_id_users") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_files")), - sa.UniqueConstraint("file_id", "journalist_id", name=op.f("uq_seen_files_file_id")), - ) - op.create_table( - "seen_messages", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("message_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_seen_messages_journalist_id_users") - ), - sa.ForeignKeyConstraint( - ["message_id"], ["messages.id"], name=op.f("fk_seen_messages_message_id_messages") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_messages")), - sa.UniqueConstraint( - "message_id", "journalist_id", name=op.f("uq_seen_messages_message_id") - ), - ) - op.create_table( - "seen_replies", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("reply_id", sa.Integer(), nullable=False), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_seen_replies_journalist_id_users") - ), - sa.ForeignKeyConstraint( - ["reply_id"], ["replies.id"], name=op.f("fk_seen_replies_reply_id_replies") - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_replies")), - sa.UniqueConstraint("reply_id", "journalist_id", name=op.f("uq_seen_replies_reply_id")), - ) - - -def downgrade(): - op.drop_table("seen_replies") - op.drop_table("seen_messages") - op.drop_table("seen_files") diff --git a/alembic/versions/d7c8af95bc8e_initial.py b/alembic/versions/d7c8af95bc8e_initial.py new file mode 100644 index 000000000..3b39bcc20 --- /dev/null +++ b/alembic/versions/d7c8af95bc8e_initial.py @@ -0,0 +1,321 @@ +"""Initial schema, squashed from old head 9ba8d7524871 (per #1500). + +Revision ID: d7c8af95bc8e +Revises: +Create Date: 2022-06-09 16:41:11.913336 + +""" +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d7c8af95bc8e" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "sources", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("journalist_designation", sa.String(length=255), nullable=False), + sa.Column("document_count", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column( + "is_flagged", sa.Boolean(name="is_flagged"), server_default=sa.text("0"), nullable=True + ), + sa.Column("public_key", sa.Text(), nullable=True), + sa.Column("fingerprint", sa.String(length=64), nullable=True), + sa.Column("interaction_count", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column( + "is_starred", sa.Boolean(name="is_starred"), server_default=sa.text("0"), nullable=True + ), + sa.Column("last_updated", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_sources")), + sa.UniqueConstraint("uuid", name=op.f("uq_sources_uuid")), + ) + op.create_table( + "users", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("username", sa.String(length=255), nullable=False), + sa.Column("firstname", sa.String(length=64), nullable=True), + sa.Column("lastname", sa.String(length=64), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_users")), + sa.UniqueConstraint("uuid", name=op.f("uq_users_uuid")), + ) + op.create_table( + "replysendstatuses", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_replysendstatuses")), + sa.UniqueConstraint("name", name=op.f("uq_replysendstatuses_name")), + ) + op.create_table( + "draftreplies", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("timestamp", sa.DateTime(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("journalist_id", sa.Integer(), nullable=True), + sa.Column("file_counter", sa.Integer(), nullable=False), + sa.Column("content", sa.Text(), nullable=True), + sa.Column("send_status_id", sa.Integer(), nullable=True), + sa.UniqueConstraint("uuid", name=op.f("uq_draftreplies_uuid")), + sa.ForeignKeyConstraint( + ["source_id"], ["sources.id"], name=op.f("fk_draftreplies_source_id_sources") + ), + sa.ForeignKeyConstraint( + ["journalist_id"], ["users.id"], name=op.f("fk_draftreplies_journalist_id_users") + ), + sa.ForeignKeyConstraint( + ["send_status_id"], + ["replysendstatuses.id"], + name=op.f("fk_draftreplies_send_status_id_replysendstatuses"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_draftreplies")), + ) + op.create_table( + "downloaderrors", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_downloaderrors")), + sa.UniqueConstraint("name", name=op.f("uq_downloaderrors_name")), + ) + op.create_table( + "files", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("file_counter", sa.Integer(), nullable=False), + sa.Column("size", sa.Integer(), nullable=False), + sa.Column("download_url", sa.String(length=255), nullable=False), + sa.Column( + "is_downloaded", + sa.Boolean(name="is_downloaded"), + server_default=sa.text("0"), + nullable=False, + ), + sa.Column( + "is_decrypted", + sa.Boolean(name="is_decrypted"), + sa.CheckConstraint( + "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", + name="files_compare_is_downloaded_vs_is_decrypted", + ), + nullable=True, + ), + sa.Column("download_error_id", sa.Integer(), nullable=True), + sa.Column( + "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False + ), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("last_updated", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_files")), + sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), + sa.UniqueConstraint("uuid", name=op.f("uq_files_uuid")), + sa.ForeignKeyConstraint( + ["download_error_id"], + ["downloaderrors.id"], + name=op.f("fk_files_download_error_id_downloaderrors"), + ), + sa.ForeignKeyConstraint( + ["source_id"], ["sources.id"], name=op.f("fk_files_source_id_sources") + ), + ) + op.create_table( + "messages", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("file_counter", sa.Integer(), nullable=False), + sa.Column("size", sa.Integer(), nullable=False), + sa.Column("download_url", sa.String(length=255), nullable=False), + sa.Column( + "is_downloaded", + sa.Boolean(name="is_downloaded"), + server_default=sa.text("0"), + nullable=False, + ), + sa.Column( + "is_decrypted", + sa.Boolean(name="is_decrypted"), + sa.CheckConstraint( + "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", + name="messages_compare_is_downloaded_vs_is_decrypted", + ), + nullable=True, + ), + sa.Column("download_error_id", sa.Integer(), nullable=True), + sa.Column( + "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False + ), + sa.Column( + "content", + sa.Text(), + sa.CheckConstraint( + "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", + name="ck_message_compare_download_vs_content", + ), + nullable=True, + ), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("last_updated", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), + sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), + sa.UniqueConstraint("uuid", name=op.f("uq_messages_uuid")), + sa.ForeignKeyConstraint( + ["download_error_id"], + ["downloaderrors.id"], + name=op.f("fk_messages_download_error_id_downloaderrors"), + ), + sa.ForeignKeyConstraint( + ["source_id"], ["sources.id"], name=op.f("fk_messages_source_id_sources") + ), + ) + op.create_table( + "replies", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("filename", sa.String(length=255), nullable=False), + sa.Column("file_counter", sa.Integer(), nullable=False), + sa.Column("size", sa.Integer(), nullable=True), + sa.Column( + "content", + sa.Text(), + sa.CheckConstraint( + "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", + name="replies_compare_download_vs_content", + ), + nullable=True, + ), + sa.Column( + "is_decrypted", + sa.Boolean(name="is_decrypted"), + sa.CheckConstraint( + "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", + name="replies_compare_is_downloaded_vs_is_decrypted", + ), + nullable=True, + ), + sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), + sa.Column("download_error_id", sa.Integer(), nullable=True), + sa.Column("journalist_id", sa.Integer(), nullable=True), + sa.Column("last_updated", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint( + ["download_error_id"], + ["downloaderrors.id"], + name=op.f("fk_replies_download_error_id_downloaderrors"), + ), + sa.ForeignKeyConstraint( + ["journalist_id"], ["users.id"], name=op.f("fk_replies_journalist_id_users") + ), + sa.ForeignKeyConstraint( + ["source_id"], ["sources.id"], name=op.f("fk_replies_source_id_sources") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_replies")), + sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), + sa.UniqueConstraint("uuid", name=op.f("uq_replies_uuid")), + ) + op.create_table( + "seen_files", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("file_id", sa.Integer(), nullable=False), + sa.Column("journalist_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["file_id"], ["files.id"], name=op.f("fk_seen_files_file_id_files") + ), + sa.ForeignKeyConstraint( + ["journalist_id"], ["users.id"], name=op.f("fk_seen_files_journalist_id_users") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_files")), + sa.UniqueConstraint("file_id", "journalist_id", name=op.f("uq_seen_files_file_id")), + ) + op.create_table( + "seen_messages", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("message_id", sa.Integer(), nullable=False), + sa.Column("journalist_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["journalist_id"], ["users.id"], name=op.f("fk_seen_messages_journalist_id_users") + ), + sa.ForeignKeyConstraint( + ["message_id"], ["messages.id"], name=op.f("fk_seen_messages_message_id_messages") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_messages")), + sa.UniqueConstraint( + "message_id", "journalist_id", name=op.f("uq_seen_messages_message_id") + ), + ) + op.create_table( + "seen_replies", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("reply_id", sa.Integer(), nullable=False), + sa.Column("journalist_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["journalist_id"], ["users.id"], name=op.f("fk_seen_replies_journalist_id_users") + ), + sa.ForeignKeyConstraint( + ["reply_id"], ["replies.id"], name=op.f("fk_seen_replies_reply_id_replies") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_seen_replies")), + sa.UniqueConstraint("reply_id", "journalist_id", name=op.f("uq_seen_replies_reply_id")), + ) + op.create_table( + "deletedconversation", + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedconversation")), + ) + op.create_table( + "deletedsource", + sa.Column("uuid", sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedsource")), + ) + # ### end Alembic commands ### + + # Set enum values: + conn = op.get_bind() + + # downloaderrors (from original Alembic version 7f682532afa2) + conn.execute( + """ + INSERT INTO downloaderrors + ('name') + VALUES + ('CHECKSUM_ERROR'), + ('DECRYPTION_ERROR'); + """ + ) + + # replysendstatuses (from original Alembic version 86b01b6290da) + conn.execute( + """ + INSERT INTO replysendstatuses + ('name') + VALUES + ('PENDING'), + ('FAILED'); + """ + ) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("seen_replies") + op.drop_table("seen_messages") + op.drop_table("seen_files") + op.drop_table("replies") + op.drop_table("messages") + op.drop_table("files") + op.drop_table("draftreplies") + op.drop_table("users") + op.drop_table("sources") + op.drop_table("replysendstatuses") + op.drop_table("downloaderrors") + op.drop_table("deletedsource") + op.drop_table("deletedconversation") + # ### end Alembic commands ### diff --git a/alembic/versions/eff1387cfd0b_add_deletedconversation_table.py b/alembic/versions/eff1387cfd0b_add_deletedconversation_table.py deleted file mode 100644 index 0ad578e25..000000000 --- a/alembic/versions/eff1387cfd0b_add_deletedconversation_table.py +++ /dev/null @@ -1,34 +0,0 @@ -"""add deletedconversation table - -Revision ID: eff1387cfd0b -Revises: bd57477f19a2 -Create Date: 2022-02-24 13:11:22.227528 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "eff1387cfd0b" -down_revision = "bd57477f19a2" -branch_labels = None -depends_on = None - - -def upgrade(): - # Add deletedconversation table to manage locally-deleted records - # and ensure they do not get re-downloaded to the database during - # a network race condition. - # UUID was chosen as PK to avoid storing data such as source_id that - # could divulge information about the source account creation timeline. - # Note that records in this table are purged every 15 seconds. - op.create_table( - "deletedconversation", - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.PrimaryKeyConstraint("uuid", name=op.f("pk_deletedconversation")), - ) - - -def downgrade(): - op.drop_table("deletedconversation") diff --git a/alembic/versions/fb657f2ee8a7_drop_file_original_filename.py b/alembic/versions/fb657f2ee8a7_drop_file_original_filename.py deleted file mode 100644 index a09c7533c..000000000 --- a/alembic/versions/fb657f2ee8a7_drop_file_original_filename.py +++ /dev/null @@ -1,72 +0,0 @@ -"""drop File.original_filename - -Revision ID: fb657f2ee8a7 -Revises: 86b01b6290da -Create Date: 2020-01-23 18:55:09.857324 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "fb657f2ee8a7" -down_revision = "86b01b6290da" -branch_labels = None -depends_on = None - - -def upgrade(): - conn = op.get_bind() - - op.rename_table("files", "original_files") - - conn.execute( - """ - CREATE TABLE files ( - id INTEGER NOT NULL, - uuid VARCHAR(36) NOT NULL, - filename VARCHAR(255) NOT NULL, - file_counter INTEGER NOT NULL, - size INTEGER NOT NULL, - download_url VARCHAR(255) NOT NULL, - is_downloaded BOOLEAN DEFAULT 0 NOT NULL, - is_read BOOLEAN DEFAULT 0 NOT NULL, - is_decrypted BOOLEAN, - source_id INTEGER NOT NULL, - CONSTRAINT pk_files PRIMARY KEY (id), - CONSTRAINT fk_files_source_id_sources FOREIGN KEY(source_id) REFERENCES sources (id), - CONSTRAINT uq_messages_source_id_file_counter UNIQUE (source_id, file_counter), - CONSTRAINT uq_files_uuid UNIQUE (uuid), - CONSTRAINT files_compare_is_downloaded_vs_is_decrypted CHECK (CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END), - CONSTRAINT ck_files_is_downloaded CHECK (is_downloaded IN (0, 1)), - CONSTRAINT ck_files_is_read CHECK (is_read IN (0, 1)), - CONSTRAINT ck_files_is_decrypted CHECK (is_decrypted IN (0, 1)) - ) - """ - ) - - conn.execute( - """ - INSERT INTO files - (id, uuid, filename, file_counter, size, download_url, is_downloaded, - is_decrypted, is_read, source_id) - SELECT id, uuid, filename, file_counter, size, download_url, is_downloaded, - is_decrypted, is_read, source_id - FROM original_files - """ - ) - - op.drop_table("original_files") - - -def downgrade(): - op.add_column( - "files", - sa.Column( - "original_filename", - sa.VARCHAR(length=255), - server_default=sa.text("''"), - nullable=False, - ), - ) diff --git a/alembic/versions/fecf1191b6f0_remove_decryption_vs_content_contraint.py b/alembic/versions/fecf1191b6f0_remove_decryption_vs_content_contraint.py deleted file mode 100644 index f611da5ae..000000000 --- a/alembic/versions/fecf1191b6f0_remove_decryption_vs_content_contraint.py +++ /dev/null @@ -1,218 +0,0 @@ -"""remove decryption_vs_content contraint - -Revision ID: fecf1191b6f0 -Revises: 2f363b3d680e -Create Date: 2019-06-18 19:14:17.862910 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = "fecf1191b6f0" -down_revision = "2f363b3d680e" -branch_labels = None -depends_on = None - - -def upgrade(): - # Save existing tables we want to modify. - op.rename_table("messages", "messages_tmp") - op.rename_table("replies", "replies_tmp") - - # Create new tables without the constraint. - op.create_table( - "messages", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=False), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column( - "is_downloaded", - sa.Boolean(name="is_downloaded"), - server_default=sa.text("0"), - nullable=False, - ), - sa.Column( - "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False - ), - sa.Column("download_url", sa.String(length=255), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_messages_uuid")), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_messages_source_id_sources") - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name=op.f("ck_message_compare_download_vs_content"), - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="messages_compare_is_downloaded_vs_is_decrypted", - ), - ) - - op.create_table( - "replies", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=True), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_replies")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_replies_uuid")), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_replies_source_id_sources") - ), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_replies_journalist_id_users") - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name="replies_compare_download_vs_content", - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="replies_compare_is_downloaded_vs_is_decrypted", - ), - ) - - # Copy existing data into new tables. - conn = op.get_bind() - conn.execute( - """ - INSERT INTO messages - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, is_read, download_url - FROM messages_tmp - """ - ) - conn.execute( - """ - INSERT INTO replies - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, journalist_id - FROM replies_tmp - """ - ) - - # Delete the old tables. - op.drop_table("messages_tmp") - op.drop_table("replies_tmp") - - -def downgrade(): - # Save existing tables we want to modify. - op.rename_table("messages", "messages_tmp") - op.rename_table("replies", "replies_tmp") - - # Create new tables with the constraint. - op.create_table( - "messages", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=False), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column( - "is_downloaded", - sa.Boolean(name="is_downloaded"), - server_default=sa.text("0"), - nullable=False, - ), - sa.Column( - "is_read", sa.Boolean(name="is_read"), server_default=sa.text("0"), nullable=False - ), - sa.Column("download_url", sa.String(length=255), nullable=False), - sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_messages_uuid")), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_messages_source_id_sources") - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name=op.f("ck_message_compare_download_vs_content"), - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="messages_compare_is_downloaded_vs_is_decrypted", - ), - sa.CheckConstraint( - "CASE WHEN is_decrypted = 0 THEN content IS NULL ELSE 1 END", - name="messages_compare_is_decrypted_vs_content", - ), - ) - - op.create_table( - "replies", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("uuid", sa.String(length=36), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("filename", sa.String(length=255), nullable=False), - sa.Column("file_counter", sa.Integer(), nullable=False), - sa.Column("size", sa.Integer(), nullable=True), - sa.Column("content", sa.Text(), nullable=True), - sa.Column("is_decrypted", sa.Boolean(name="is_decrypted"), nullable=True), - sa.Column("is_downloaded", sa.Boolean(name="is_downloaded"), nullable=True), - sa.Column("journalist_id", sa.Integer(), nullable=True), - sa.PrimaryKeyConstraint("id", name=op.f("pk_replies")), - sa.UniqueConstraint("source_id", "file_counter", name="uq_messages_source_id_file_counter"), - sa.UniqueConstraint("uuid", name=op.f("uq_replies_uuid")), - sa.ForeignKeyConstraint( - ["source_id"], ["sources.id"], name=op.f("fk_replies_source_id_sources") - ), - sa.ForeignKeyConstraint( - ["journalist_id"], ["users.id"], name=op.f("fk_replies_journalist_id_users") - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN content IS NULL ELSE 1 END", - name="replies_compare_download_vs_content", - ), - sa.CheckConstraint( - "CASE WHEN is_downloaded = 0 THEN is_decrypted IS NULL ELSE 1 END", - name="replies_compare_is_downloaded_vs_is_decrypted", - ), - sa.CheckConstraint( - "CASE WHEN is_decrypted = 0 THEN content IS NULL ELSE 1 END", - name="replies_compare_is_decrypted_vs_content", - ), - ) - - # Copy existing data into new tables. - conn = op.get_bind() - conn.execute( - """ - INSERT INTO messages - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, download_url, is_read - FROM messages_tmp - """ - ) - conn.execute( - """ - INSERT INTO replies - SELECT id, uuid, source_id, filename, file_counter, size, content, is_decrypted, - is_downloaded, journalist_id - FROM replies_tmp - """ - ) - - # Delete the old tables. - op.drop_table("messages_tmp") - op.drop_table("replies_tmp") diff --git a/tests/migrations/test_a4bf1f58ce69.py b/tests/migrations/test_a4bf1f58ce69.py deleted file mode 100644 index 84c60d09a..000000000 --- a/tests/migrations/test_a4bf1f58ce69.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import random -import subprocess - -from securedrop_client import db -from securedrop_client.db import DraftReply, Reply, User - -from .utils import add_draft_reply, add_reply, add_source, add_user - -random.seed("=^..^=..^=..^=") - - -class UpgradeTester: - """ - Verify that upgrading to the target migration results in the replacement of uuid with id of the - user in the replies table's journalist_id column. - """ - - NUM_USERS = 20 - NUM_SOURCES = 20 - NUM_REPLIES = 40 - - def __init__(self, homedir): - subprocess.check_call(["sqlite3", os.path.join(homedir, "svs.sqlite"), ".databases"]) - self.session = db.make_session_maker(homedir)() - - def load_data(self): - """ - Load data that has the bug where user.uuid is stored in replies.journalist_id and - draftreplies.journalist_id. - """ - for _ in range(self.NUM_SOURCES): - add_source(self.session) - - self.session.commit() - - for i in range(self.NUM_USERS): - if i == 0: - # As of this migration, the server tells the client that the associated journalist - # of a reply has been deleted by returning "deleted" as the uuid of the associated - # journalist. This gets stored as the jouranlist_id in the replies table. - # - # Make sure to test this case as well. - add_user(self.session, "deleted") - source_id = random.randint(1, self.NUM_SOURCES) - add_reply(self.session, "deleted", source_id) - else: - add_user(self.session) - - self.session.commit() - - # Add replies from randomly-selected journalists to a randomly-selected sources - for _ in range(1, self.NUM_REPLIES): - journalist_id = random.randint(1, self.NUM_USERS) - journalist = self.session.query(User).filter_by(id=journalist_id).one() - source_id = random.randint(1, self.NUM_SOURCES) - add_reply(self.session, journalist.uuid, source_id) - - # Add draft replies from randomly-selected journalists to a randomly-selected sources - for _ in range(1, self.NUM_REPLIES): - journalist_id = random.randint(1, self.NUM_USERS) - journalist = self.session.query(User).filter_by(id=journalist_id).one() - source_id = random.randint(1, self.NUM_SOURCES) - add_draft_reply(self.session, journalist.uuid, source_id) - - self.session.commit() - - def check_upgrade(self): - """ - Make sure each reply in the replies and draftreplies tables have the correct journalist_id - stored for the associated journalist by making sure a User account exists with that - journalist id. - """ - replies = self.session.query(Reply).all() - assert len(replies) - - for reply in replies: - # Will fail if User does not exist - self.session.query(User).filter_by(id=reply.journalist_id).one() - - draftreplies = self.session.query(DraftReply).all() - assert len(draftreplies) - - for draftreply in draftreplies: - # Will fail if User does not exist - self.session.query(User).filter_by(id=draftreply.journalist_id).one() - - self.session.close() - - -class DowngradeTester: - """ - Nothing to test since the downgrade path doesn't do anything. - """ - - def __init__(self, homedir): - pass - - def load_data(self): - pass - - def check_downgrade(self): - pass diff --git a/tests/migrations/test_bd57477f19a2.py b/tests/migrations/test_bd57477f19a2.py deleted file mode 100644 index 0bb3b0229..000000000 --- a/tests/migrations/test_bd57477f19a2.py +++ /dev/null @@ -1,235 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import random -import subprocess - -import pytest -from sqlalchemy import text -from sqlalchemy.exc import IntegrityError - -from securedrop_client import db -from securedrop_client.db import Reply, User - -from .utils import ( - add_file, - add_message, - add_reply, - add_source, - add_user, - mark_file_as_seen, - mark_message_as_seen, - mark_reply_as_seen, -) - - -class UpgradeTester: - """ - Verify that upgrading to the target migration results in the creation of the seen tables. - """ - - NUM_USERS = 20 - NUM_SOURCES = 20 - NUM_REPLIES = 40 - - def __init__(self, homedir): - subprocess.check_call(["sqlite3", os.path.join(homedir, "svs.sqlite"), ".databases"]) - self.session = db.make_session_maker(homedir)() - - def load_data(self): - for source_id in range(1, self.NUM_SOURCES + 1): - add_source(self.session) - - # Add zero to a few messages from each source, some messages are set to downloaded - for _ in range(random.randint(0, 2)): - add_message(self.session, source_id) - - # Add zero to a few files from each source, some files are set to downloaded - for _ in range(random.randint(0, 2)): - add_file(self.session, source_id) - - self.session.commit() - - for i in range(self.NUM_USERS): - if i == 0: - # As of this migration, the server tells the client that the associated journalist - # of a reply has been deleted by returning "deleted" as the uuid of the associated - # journalist. This gets stored as the jouranlist_id in the replies table. - # - # Make sure to test this case as well. - add_user(self.session, "deleted") - source_id = random.randint(1, self.NUM_SOURCES) - user = self.session.query(User).filter_by(uuid="deleted").one() - add_reply(self.session, user.id, source_id) - else: - add_user(self.session) - - self.session.commit() - - # Add replies from randomly-selected journalists to a randomly-selected sources - for _ in range(1, self.NUM_REPLIES): - journalist_id = random.randint(1, self.NUM_USERS) - source_id = random.randint(1, self.NUM_SOURCES) - add_reply(self.session, journalist_id, source_id) - - self.session.commit() - - def check_upgrade(self): - """ - Make sure seen tables exist and work as expected. - """ - replies = self.session.query(Reply).all() - assert len(replies) - - for reply in replies: - # Will fail if User does not exist - self.session.query(User).filter_by(id=reply.journalist_id).one() - - sql = "SELECT * FROM files" - files = self.session.execute(text(sql)).fetchall() - - sql = "SELECT * FROM messages" - messages = self.session.execute(text(sql)).fetchall() - - sql = "SELECT * FROM replies" - replies = self.session.execute(text(sql)).fetchall() - - # Now seen tables exist, so you should be able to mark some files, messages, and replies - # as seen - for file in files: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_file_as_seen(self.session, file.id, selected_journo_id) - for message in messages: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_message_as_seen(self.session, message.id, selected_journo_id) - for reply in replies: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_reply_as_seen(self.session, reply.id, selected_journo_id) - - # Check unique constraint on (reply_id, journalist_id) - params = {"reply_id": 100, "journalist_id": 100} - sql = """ - INSERT INTO seen_replies (reply_id, journalist_id) - VALUES (:reply_id, :journalist_id); - """ - self.session.execute(text(sql), params) - with pytest.raises(IntegrityError): - self.session.execute(text(sql), params) - - # Check unique constraint on (message_id, journalist_id) - params = {"message_id": 100, "journalist_id": 100} - sql = """ - INSERT INTO seen_messages (message_id, journalist_id) - VALUES (:message_id, :journalist_id); - """ - self.session.execute(text(sql), params) - with pytest.raises(IntegrityError): - self.session.execute(text(sql), params) - - # Check unique constraint on (file_id, journalist_id) - params = {"file_id": 101, "journalist_id": 100} - sql = """ - INSERT INTO seen_files (file_id, journalist_id) - VALUES (:file_id, :journalist_id); - """ - self.session.execute(text(sql), params) - with pytest.raises(IntegrityError): - self.session.execute(text(sql), params) - - -class DowngradeTester: - """ - Verify that downgrading from the target migration keeps in place the updates from the migration - since there is no need to add bad data back into the db (the migration is backwards compatible). - """ - - NUM_USERS = 20 - NUM_SOURCES = 20 - NUM_REPLIES = 40 - - def __init__(self, homedir): - subprocess.check_call(["sqlite3", os.path.join(homedir, "svs.sqlite"), ".databases"]) - self.session = db.make_session_maker(homedir)() - - def load_data(self): - for source_id in range(1, self.NUM_SOURCES + 1): - add_source(self.session) - - # Add zero to a few messages from each source, some messages are set to downloaded - for _ in range(random.randint(0, 3)): - add_message(self.session, source_id) - - # Add zero to a few files from each source, some files are set to downloaded - for _ in range(random.randint(0, 3)): - add_file(self.session, source_id) - - self.session.commit() - - for i in range(self.NUM_USERS): - if i == 0: - # As of this migration, the server tells the client that the associated journalist - # of a reply has been deleted by returning "deleted" as the uuid of the associated - # journalist. This gets stored as the jouranlist_id in the replies table. - # - # Make sure to test this case as well. - add_user(self.session, "deleted") - source_id = random.randint(1, self.NUM_SOURCES) - add_reply(self.session, "deleted", source_id) - else: - add_user(self.session) - - self.session.commit() - - # Add replies from randomly-selected journalists to a randomly-selected sources - for _ in range(1, self.NUM_REPLIES): - journalist_id = random.randint(1, self.NUM_USERS) - source_id = random.randint(1, self.NUM_SOURCES) - add_reply(self.session, journalist_id, source_id) - - self.session.commit() - - # Mark some files, messages, and replies as seen - sql = "SELECT * FROM files" - files = self.session.execute(text(sql)).fetchall() - for file in files: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_file_as_seen(self.session, file.id, selected_journo_id) - - sql = "SELECT * FROM messages" - messages = self.session.execute(text(sql)).fetchall() - for message in messages: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_message_as_seen(self.session, message.id, selected_journo_id) - - sql = "SELECT * FROM replies" - replies = self.session.execute(text(sql)).fetchall() - for reply in replies: - if random.choice([0, 1]): - selected_journo_id = random.randint(1, self.NUM_USERS) - mark_reply_as_seen(self.session, reply.id, selected_journo_id) - - self.session.commit() - - def check_downgrade(self): - """ - Check that seen tables no longer exist. - """ - params = {"table_name": "seen_files"} - sql = "SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;" - seen_files_exists = self.session.execute(text(sql), params).fetchall() - assert not seen_files_exists - - params = {"table_name": "seen_messages"} - sql = "SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;" - seen_messages_exists = self.session.execute(text(sql), params).fetchall() - assert not seen_messages_exists - - params = {"table_name": "seen_replies"} - sql = "SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;" - seen_replies_exists = self.session.execute(text(sql), params).fetchall() - assert not seen_replies_exists diff --git a/tests/migrations/test_d7c8af95bc8e.py b/tests/migrations/test_d7c8af95bc8e.py new file mode 100644 index 000000000..59e99b768 --- /dev/null +++ b/tests/migrations/test_d7c8af95bc8e.py @@ -0,0 +1,47 @@ +import os +import subprocess + +import pytest +from sqlalchemy import text +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import scoped_session + +ENUMS = { + "downloaderrors": ["CHECKSUM_ERROR", "DECRYPTION_ERROR"], + "replysendstatuses": ["PENDING", "FAILED"], +} + + +class UpgradeTester: + """Enum tables exist and contains the expected values.""" + + def __init__(self, homedir: str, session: scoped_session) -> None: + subprocess.check_call(["sqlite3", os.path.join(homedir, "svs.sqlite"), ".databases"]) + self.session = session + + def load_data(self): + pass + + def check_upgrade(self): + for table, values in ENUMS.items(): + for value in values: + result = self.session.execute( + text(f"SELECT * FROM {table} WHERE name = '{value}'") + ).fetchone() + assert result is not None + + +class DowngradeTester: + """Enum tables do not exist.""" + + def __init__(self, homedir: str, session: scoped_session) -> None: + subprocess.check_call(["sqlite3", os.path.join(homedir, "svs.sqlite"), ".databases"]) + self.session = session + + def load_data(self): + pass + + def check_downgrade(self): + for table in ENUMS: + with pytest.raises(OperationalError): + self.session.execute(text(f"SELECT name FROM {table}")) diff --git a/tests/migrations/utils.py b/tests/migrations/utils.py deleted file mode 100644 index dc89bdafa..000000000 --- a/tests/migrations/utils.py +++ /dev/null @@ -1,327 +0,0 @@ -# -*- coding: utf-8 -*- -import random -import string -from datetime import datetime -from typing import Optional -from uuid import uuid4 - -from sqlalchemy import text -from sqlalchemy.orm.session import Session - -from securedrop_client.db import DownloadError, ReplySendStatus, Source - -random.seed("ᕕ( ᐛ )ᕗ") - - -def random_bool() -> bool: - return bool(random.getrandbits(1)) - - -def bool_or_none() -> Optional[bool]: - return random.choice([None, True, False]) - - -def random_name() -> str: - len = random.randint(1, 100) - return random_chars(len) - - -def random_username() -> str: - len = random.randint(3, 64) - return random_chars(len) - - -def random_chars(len: int, chars: str = string.printable) -> str: - return "".join([random.choice(chars) for _ in range(len)]) - - -def random_ascii_chars(len: int, chars: str = string.ascii_lowercase): - return "".join([random.choice(chars) for _ in range(len)]) - - -def random_datetime(nullable: bool = False): - if nullable and random_bool(): - return None - else: - return datetime( - year=random.randint(1, 9999), - month=random.randint(1, 12), - day=random.randint(1, 28), - hour=random.randint(0, 23), - minute=random.randint(0, 59), - second=random.randint(0, 59), - microsecond=random.randint(0, 1000), - ) - - -def add_source(session: Session) -> None: - params = { - "uuid": str(uuid4()), - "journalist_designation": random_chars(50), - "last_updated": random_datetime(nullable=True), - "interaction_count": random.randint(0, 1000), - } - sql = """ - INSERT INTO sources ( - uuid, - journalist_designation, - last_updated, - interaction_count - ) - VALUES ( - :uuid, - :journalist_designation, - :last_updated, - :interaction_count - ) - """ - session.execute(text(sql), params) - - -def add_user(session: Session, uuid: Optional[str] = None) -> None: - if not uuid: - journalist_uuid = str(uuid4()) - else: - journalist_uuid = uuid - - params = {"uuid": journalist_uuid, "username": random_username()} - sql = """ - INSERT INTO users (uuid, username) - VALUES (:uuid, :username) - """ - session.execute(text(sql), params) - - -def add_file(session: Session, source_id: int) -> None: - is_downloaded = random_bool() - is_decrypted = random_bool() if is_downloaded else None - - source = session.query(Source).filter_by(id=source_id).one() - file_counter = len(source.collection) + 1 - - params = { - "uuid": str(uuid4()), - "source_id": source_id, - "filename": random_chars(50) + "-doc.gz.gpg", - "file_counter": file_counter, - "size": random.randint(0, 1024 * 1024 * 500), - "download_url": random_chars(50), - "is_downloaded": is_downloaded, - "is_decrypted": is_decrypted, - "is_read": random.choice([True, False]), - "last_updated": random_datetime(), - } - sql = """ - INSERT INTO files - ( - uuid, - source_id, - filename, - file_counter, - size, - download_url, - is_downloaded, - is_decrypted, - is_read, - last_updated - ) - VALUES - ( - :uuid, - :source_id, - :filename, - :file_counter, - :size, - :download_url, - :is_downloaded, - :is_decrypted, - :is_read, - :last_updated - ) - """ - session.execute(text(sql), params) - - -def add_message(session: Session, source_id: int) -> None: - is_downloaded = random_bool() - is_decrypted = random_bool() if is_downloaded else None - - content = random_chars(1000) if is_downloaded else None - - source = session.query(Source).filter_by(id=source_id).one() - file_counter = len(source.collection) + 1 - - params = { - "uuid": str(uuid4()), - "source_id": source_id, - "filename": random_chars(50) + "-doc.gz.gpg", - "file_counter": file_counter, - "size": random.randint(0, 1024 * 1024 * 500), - "content": content, - "download_url": random_chars(50), - "is_downloaded": is_downloaded, - "is_decrypted": is_decrypted, - "is_read": random.choice([True, False]), - "last_updated": random_datetime(), - } - sql = """ - INSERT INTO messages - ( - uuid, - source_id, - filename, - file_counter, - size, - content, - download_url, - is_downloaded, - is_decrypted, - is_read, - last_updated - ) - VALUES - ( - :uuid, - :source_id, - :filename, - :file_counter, - :size, - :content, - :download_url, - :is_downloaded, - :is_decrypted, - :is_read, - :last_updated - ) - """ - session.execute(text(sql), params) - - -def add_reply(session: Session, journalist_id: int, source_id: int) -> None: - is_downloaded = random_bool() if random_bool() else None - is_decrypted = random_bool() if is_downloaded else None - - download_errors = session.query(DownloadError).all() - download_error_ids = [error.id for error in download_errors] - - content = random_chars(1000) if is_downloaded else None - - source = session.query(Source).filter_by(id=source_id).one() - file_counter = len(source.collection) + 1 - - params = { - "uuid": str(uuid4()), - "journalist_id": journalist_id, - "source_id": source_id, - "filename": random_chars(50) + "-reply.gpg", - "file_counter": file_counter, - "size": random.randint(0, 1024 * 1024 * 500), - "content": content, - "is_downloaded": is_downloaded, - "is_decrypted": is_decrypted, - "download_error_id": random.choice(download_error_ids), - "last_updated": random_datetime(), - } - sql = """ - INSERT INTO replies - ( - uuid, - journalist_id, - source_id, - filename, - file_counter, - size, - content, - is_downloaded, - is_decrypted, - download_error_id, - last_updated - ) - VALUES - ( - :uuid, - :journalist_id, - :source_id, - :filename, - :file_counter, - :size, - :content, - :is_downloaded, - :is_decrypted, - :download_error_id, - :last_updated - ) - """ - session.execute(text(sql), params) - - -def mark_file_as_seen(session: Session, file_id: int, journalist_id: int) -> None: - params = {"file_id": file_id, "journalist_id": journalist_id} - sql = """ - INSERT INTO seen_files (file_id, journalist_id) - VALUES (:file_id, :journalist_id) - """ - session.execute(text(sql), params) - - -def mark_message_as_seen(session: Session, message_id: int, journalist_id: int) -> None: - params = {"message_id": message_id, "journalist_id": journalist_id} - sql = """ - INSERT INTO seen_messages (message_id, journalist_id) - VALUES (:message_id, :journalist_id) - """ - session.execute(text(sql), params) - - -def mark_reply_as_seen(session: Session, reply_id: int, journalist_id: int): - params = {"reply_id": reply_id, "journalist_id": journalist_id} - sql = """ - INSERT INTO seen_replies (reply_id, journalist_id) - VALUES (:reply_id, :journalist_id) - """ - session.execute(text(sql), params) - - -def add_draft_reply(session: Session, journalist_id: int, source_id: int) -> None: - reply_send_statuses = session.query(ReplySendStatus).all() - reply_send_status_ids = [reply_send_status.id for reply_send_status in reply_send_statuses] - - content = random_chars(1000) - - source = session.query(Source).filter_by(id=source_id).one() - - file_counter = len(source.collection) + 1 - - params = { - "uuid": str(uuid4()), - "journalist_id": journalist_id, - "source_id": source_id, - "file_counter": file_counter, - "content": content, - "send_status_id": random.choice(reply_send_status_ids), - "timestamp": random_datetime(), - } - - sql = """ - INSERT INTO draftreplies - ( - uuid, - journalist_id, - source_id, - file_counter, - content, - send_status_id, - timestamp - ) - VALUES - ( - :uuid, - :journalist_id, - :source_id, - :file_counter, - :content, - :send_status_id, - :timestamp - ) - """ - session.execute(text(sql), params) diff --git a/tests/test_alembic.py b/tests/test_alembic.py index d3a8cdc4f..487a324f3 100644 --- a/tests/test_alembic.py +++ b/tests/test_alembic.py @@ -6,11 +6,12 @@ from os import path import pytest -from sqlalchemy import text +from sqlalchemy import create_engine, text +from sqlalchemy.orm import scoped_session, sessionmaker from alembic.config import Config as AlembicConfig from alembic.script import ScriptDirectory -from securedrop_client.db import Base, convention, make_session_maker +from securedrop_client.db import Base, convention from . import conftest @@ -20,11 +21,22 @@ x.split(".")[0].split("_")[0] for x in os.listdir(MIGRATION_PATH) if x.endswith(".py") ] -DATA_MIGRATIONS = ["a4bf1f58ce69", "bd57477f19a2"] +DATA_MIGRATIONS = ["d7c8af95bc8e"] WHITESPACE_REGEX = re.compile(r"\s+") +def make_session_maker(home: str) -> scoped_session: + """ + Duplicate securedrop_client.db.make_session_maker so that data migrations are decoupled + from that implementation. + """ + db_path = os.path.join(home, "svs.sqlite") + engine = create_engine("sqlite:///{}".format(db_path)) + maker = sessionmaker(bind=engine) + return scoped_session(maker) + + def list_migrations(cfg_path, head): cfg = AlembicConfig(cfg_path) script = ScriptDirectory.from_config(cfg) @@ -140,16 +152,17 @@ def test_alembic_migration_upgrade(alembic_config, config, migration): @pytest.mark.parametrize("migration", DATA_MIGRATIONS) def test_alembic_migration_upgrade_with_data(alembic_config, config, migration, homedir): """ - Upgrade to one migration before the target migration, load data, then upgrade in order to test - that the upgrade is successful when there is data. + Upgrade to one migration before the target migration (if there is one), + load data, then upgrade in order to test that the upgrade is successful + when there is data. """ migrations = list_migrations(alembic_config, migration) - if len(migrations) == 1: - return - upgrade(alembic_config, migrations[-2]) + if len(migrations) > 1: + upgrade(alembic_config, migrations[-2]) mod_name = "tests.migrations.test_{}".format(migration) mod = __import__(mod_name, fromlist=["UpgradeTester"]) - upgrade_tester = mod.UpgradeTester(homedir) + session = make_session_maker(homedir) + upgrade_tester = mod.UpgradeTester(homedir, session) upgrade_tester.load_data() upgrade(alembic_config, migration) upgrade_tester.check_upgrade() @@ -177,7 +190,8 @@ def test_alembic_migration_downgrade_with_data(alembic_config, config, migration upgrade(alembic_config, migration) mod_name = "tests.migrations.test_{}".format(migration) mod = __import__(mod_name, fromlist=["DowngradeTester"]) - downgrade_tester = mod.DowngradeTester(homedir) + session = make_session_maker(homedir) + downgrade_tester = mod.DowngradeTester(homedir, session) downgrade_tester.load_data() downgrade(alembic_config, "-1") downgrade_tester.check_downgrade()