From d3eb6050290e4b2d3c5bef943af7c68a5930b131 Mon Sep 17 00:00:00 2001 From: Kunal Mehta Date: Fri, 18 Mar 2022 15:21:25 -0400 Subject: [PATCH] Upgrade mypy and enable SQLAlchemy plugin * Upgrade mypy to 0.942 and sqlalchemy-stubs to 0.4. Type stubs are no longer bundled with mypy, so we need to explicitly install various types-{name} packages. * Since we need to install extra packages and need other dependencies like Flask, move mypy into test-requirements.txt and run it via the dev container. * The main issue spotted by the SQLAlchemy plugin, that sources.filesystem_id is nullable, is being fixed in a separate PR: #6350. * Drop the explicit type comments now that mypy can infer them. * Use `--explicit-package-bases` when running mypy, so it treats the securedrop/admin directories as modules regardless of whether `__init__.py` files exist. This is needed so the two tests/ directories don't conflict with each other. * Add missing return types to all schema changes and the templates that generate them. * In the `create_source_uuid_column` migration, drop the use of `quoted_name()`. For some reason mypy doesn't like it, but more importantly we don't need it in SQLite and we don't use it in any other schema change. * Make SecureTemporaryFile.write()'s return type match its parent by returning the number of bytes that were written. Fixes #6346. Fixes #6358. --- Makefile | 2 +- mypy.ini | 4 +- securedrop/alembic/env.py | 4 +- securedrop/alembic/script.py.mako | 4 +- .../alembic/versions/15ac9509fc68_init.py | 4 +- ..._unique_index_for_instanceconfig_valid_.py | 4 +- ...ee5bdc_added_passphrase_hash_column_to_.py | 4 +- ...c7536e8_make_journalist_id_non_nullable.py | 6 +- .../35513370ba0d_add_source_deleted_at.py | 4 +- .../3d91d6948753_create_source_uuid_column.py | 7 +- ...da3fcab826a_delete_orphaned_submissions.py | 6 +- .../versions/48a75abc0121_add_seen_tables.py | 4 +- ...fff3f969c_add_versioned_instance_config.py | 4 +- ...bb14d98_add_session_nonce_to_journalist.py | 4 +- .../versions/6db892e17271_add_reply_uuid.py | 4 +- ...e98e9_added_organization_name_field_in_.py | 4 +- .../a9fe328b053a_migrations_for_0_14_0.py | 4 +- .../b060f38c0c31_drop_source_flagged.py | 4 +- ...fdc8c_add_checksum_columns_revoke_table.py | 4 +- ...emove_partial_index_on_valid_until_set_.py | 4 +- ..._updates_journalists_otp_secret_length_.py | 4 +- ...add_column_to_track_source_deletion_of_.py | 4 +- ...ac34bb6_add_uuid_column_for_users_table.py | 4 +- .../faac8092c123_enable_security_pragmas.py | 4 +- ...f57ceef02_create_submission_uuid_column.py | 4 +- securedrop/bin/run-mypy | 17 +++++ securedrop/models.py | 44 ++++++------ .../python3/develop-requirements.in | 2 - .../python3/develop-requirements.txt | 56 --------------- .../requirements/python3/test-requirements.in | 8 +++ .../python3/test-requirements.txt | 69 +++++++++++++++++++ securedrop/secure_tempfile.py | 4 +- 32 files changed, 171 insertions(+), 134 deletions(-) create mode 100755 securedrop/bin/run-mypy diff --git a/Makefile b/Makefile index c3b598811c..35d7035362 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,7 @@ shellcheckclean: ## Clean up temporary container associated with shellcheck tar .PHONY: typelint typelint: ## Run mypy type linting. @echo "███ Running mypy type checking..." - @mypy ./securedrop ./admin + @$(SDROOT)/securedrop/bin/run-mypy @echo .PHONY: yamllint diff --git a/mypy.ini b/mypy.ini index b773724026..d20bcee4b7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -3,7 +3,9 @@ ignore_missing_imports = True no_implicit_optional = True disallow_untyped_defs = True disallow_incomplete_defs = True +warn_unused_configs = True python_version = 3.8 +plugins = sqlmypy -[mypy-tests.*] +[mypy-securedrop.tests.*,admin.tests.*] disallow_untyped_defs = False diff --git a/securedrop/alembic/env.py b/securedrop/alembic/env.py index d89e752f16..01e018db14 100644 --- a/securedrop/alembic/env.py +++ b/securedrop/alembic/env.py @@ -33,7 +33,7 @@ target_metadata = db.Model.metadata -def run_migrations_offline(): +def run_migrations_offline() -> None: """Run migrations in 'offline' mode. This configures the context with just a URL @@ -53,7 +53,7 @@ def run_migrations_offline(): context.run_migrations() -def run_migrations_online(): +def run_migrations_online() -> None: """Run migrations in 'online' mode. In this scenario we need to create an Engine diff --git a/securedrop/alembic/script.py.mako b/securedrop/alembic/script.py.mako index 2c0156303a..55df2863d2 100644 --- a/securedrop/alembic/script.py.mako +++ b/securedrop/alembic/script.py.mako @@ -16,9 +16,9 @@ branch_labels = ${repr(branch_labels)} depends_on = ${repr(depends_on)} -def upgrade(): +def upgrade() -> None: ${upgrades if upgrades else "pass"} -def downgrade(): +def downgrade() -> None: ${downgrades if downgrades else "pass"} diff --git a/securedrop/alembic/versions/15ac9509fc68_init.py b/securedrop/alembic/versions/15ac9509fc68_init.py index 2cd35cfdfc..e96bf7d042 100644 --- a/securedrop/alembic/versions/15ac9509fc68_init.py +++ b/securedrop/alembic/versions/15ac9509fc68_init.py @@ -16,7 +16,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: op.create_table( "journalists", sa.Column("id", sa.Integer(), nullable=False), @@ -84,7 +84,7 @@ def upgrade(): ) -def downgrade(): +def downgrade() -> None: op.drop_table("submissions") op.drop_table("source_stars") op.drop_table("replies") diff --git a/securedrop/alembic/versions/1ddb81fb88c2_unique_index_for_instanceconfig_valid_.py b/securedrop/alembic/versions/1ddb81fb88c2_unique_index_for_instanceconfig_valid_.py index ec65d3a49d..90e670d6d8 100644 --- a/securedrop/alembic/versions/1ddb81fb88c2_unique_index_for_instanceconfig_valid_.py +++ b/securedrop/alembic/versions/1ddb81fb88c2_unique_index_for_instanceconfig_valid_.py @@ -16,7 +16,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('instance_config', schema=None) as batch_op: batch_op.create_index('ix_one_active_instance_config', [sa.text('valid_until IS NULL')], unique=True, sqlite_where=sa.text('valid_until IS NULL')) @@ -24,7 +24,7 @@ def upgrade(): # ### end Alembic commands ### -def downgrade(): +def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('instance_config', schema=None) as batch_op: batch_op.drop_index('ix_one_active_instance_config') diff --git a/securedrop/alembic/versions/2d0ce3ee5bdc_added_passphrase_hash_column_to_.py b/securedrop/alembic/versions/2d0ce3ee5bdc_added_passphrase_hash_column_to_.py index dc2a814dfa..7f2d25cba3 100644 --- a/securedrop/alembic/versions/2d0ce3ee5bdc_added_passphrase_hash_column_to_.py +++ b/securedrop/alembic/versions/2d0ce3ee5bdc_added_passphrase_hash_column_to_.py @@ -16,11 +16,11 @@ depends_on = None -def upgrade(): +def upgrade() -> None: op.add_column("journalists", sa.Column("passphrase_hash", sa.String(length=256), nullable=True)) -def downgrade(): +def downgrade() -> None: # sqlite has no `drop column` command, so we recreate the original table # then load it from a temp table conn = op.get_bind() diff --git a/securedrop/alembic/versions/2e24fc7536e8_make_journalist_id_non_nullable.py b/securedrop/alembic/versions/2e24fc7536e8_make_journalist_id_non_nullable.py index 2af85a839b..c5fcdb140b 100644 --- a/securedrop/alembic/versions/2e24fc7536e8_make_journalist_id_non_nullable.py +++ b/securedrop/alembic/versions/2e24fc7536e8_make_journalist_id_non_nullable.py @@ -60,7 +60,7 @@ def create_deleted() -> int: return result[0][0] -def migrate_nulls(): +def migrate_nulls() -> None: """migrate existing journalist_id=NULL over to deleted or delete them""" op.execute("DELETE FROM journalist_login_attempt WHERE journalist_id IS NULL;") op.execute("DELETE FROM revoked_tokens WHERE journalist_id IS NULL;") @@ -90,7 +90,7 @@ def migrate_nulls(): op.execute(f'DELETE FROM {table} WHERE journalist_id IS NULL') # nosec -def upgrade(): +def upgrade() -> None: migrate_nulls() with op.batch_alter_table('journalist_login_attempt', schema=None) as batch_op: @@ -124,7 +124,7 @@ def upgrade(): nullable=False) -def downgrade(): +def downgrade() -> None: # We do not un-migrate the data back to journalist_id=NULL with op.batch_alter_table('seen_replies', schema=None) as batch_op: diff --git a/securedrop/alembic/versions/35513370ba0d_add_source_deleted_at.py b/securedrop/alembic/versions/35513370ba0d_add_source_deleted_at.py index 2c73e971ee..af5c64f472 100644 --- a/securedrop/alembic/versions/35513370ba0d_add_source_deleted_at.py +++ b/securedrop/alembic/versions/35513370ba0d_add_source_deleted_at.py @@ -16,7 +16,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('sources', schema=None) as batch_op: batch_op.add_column(sa.Column('deleted_at', sa.DateTime(), nullable=True)) @@ -24,7 +24,7 @@ def upgrade(): # ### end Alembic commands ### -def downgrade(): +def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('sources', schema=None) as batch_op: batch_op.drop_column('deleted_at') diff --git a/securedrop/alembic/versions/3d91d6948753_create_source_uuid_column.py b/securedrop/alembic/versions/3d91d6948753_create_source_uuid_column.py index 88b49fe08d..6161cbe2ca 100644 --- a/securedrop/alembic/versions/3d91d6948753_create_source_uuid_column.py +++ b/securedrop/alembic/versions/3d91d6948753_create_source_uuid_column.py @@ -7,7 +7,6 @@ """ from alembic import op import sqlalchemy as sa -from sqlalchemy.sql import quoted_name import uuid # revision identifiers, used by Alembic. @@ -17,7 +16,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: conn = op.get_bind() conn.execute("PRAGMA legacy_alter_table=ON") # Schema migration @@ -39,7 +38,7 @@ def upgrade(): # Now create new table with unique constraint applied. op.create_table( - quoted_name("sources", quote=False), + "sources", sa.Column("id", sa.Integer(), nullable=False), sa.Column("uuid", sa.String(length=36), nullable=False), sa.Column("filesystem_id", sa.String(length=96), nullable=True), @@ -67,6 +66,6 @@ def upgrade(): op.drop_table("sources_tmp") -def downgrade(): +def downgrade() -> None: with op.batch_alter_table("sources", schema=None) as batch_op: batch_op.drop_column("uuid") diff --git a/securedrop/alembic/versions/3da3fcab826a_delete_orphaned_submissions.py b/securedrop/alembic/versions/3da3fcab826a_delete_orphaned_submissions.py index 06b7cad96a..065283ec96 100644 --- a/securedrop/alembic/versions/3da3fcab826a_delete_orphaned_submissions.py +++ b/securedrop/alembic/versions/3da3fcab826a_delete_orphaned_submissions.py @@ -30,7 +30,7 @@ depends_on = None -def raw_sql_grab_orphaned_objects(table_name): +def raw_sql_grab_orphaned_objects(table_name: str) -> str: """Objects that have a source ID that doesn't exist in the sources table OR a NULL source ID should be deleted.""" return ('SELECT id, filename, source_id FROM {table} ' # nosec @@ -39,7 +39,7 @@ def raw_sql_grab_orphaned_objects(table_name): 'WHERE source_id IS NULL').format(table=table_name) -def upgrade(): +def upgrade() -> None: conn = op.get_bind() submissions = conn.execute( sa.text(raw_sql_grab_orphaned_objects('submissions')) @@ -100,6 +100,6 @@ def upgrade(): raise -def downgrade(): +def downgrade() -> None: # This is a destructive alembic migration, it cannot be downgraded pass diff --git a/securedrop/alembic/versions/48a75abc0121_add_seen_tables.py b/securedrop/alembic/versions/48a75abc0121_add_seen_tables.py index 59489cafbd..11108ebbb1 100644 --- a/securedrop/alembic/versions/48a75abc0121_add_seen_tables.py +++ b/securedrop/alembic/versions/48a75abc0121_add_seen_tables.py @@ -16,7 +16,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: op.create_table( "seen_files", sa.Column("id", sa.Integer(), nullable=False), @@ -51,7 +51,7 @@ def upgrade(): ) -def downgrade(): +def downgrade() -> None: op.drop_table("seen_files") op.drop_table("seen_messages") op.drop_table("seen_replies") diff --git a/securedrop/alembic/versions/523fff3f969c_add_versioned_instance_config.py b/securedrop/alembic/versions/523fff3f969c_add_versioned_instance_config.py index 0108ee17df..76284131a5 100644 --- a/securedrop/alembic/versions/523fff3f969c_add_versioned_instance_config.py +++ b/securedrop/alembic/versions/523fff3f969c_add_versioned_instance_config.py @@ -16,7 +16,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table('instance_config', sa.Column('version', sa.Integer(), nullable=False), @@ -35,7 +35,7 @@ def upgrade(): conn.execute("""INSERT INTO instance_config (allow_document_uploads) VALUES (1)""") -def downgrade(): +def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_table('instance_config') # ### end Alembic commands ### diff --git a/securedrop/alembic/versions/60f41bb14d98_add_session_nonce_to_journalist.py b/securedrop/alembic/versions/60f41bb14d98_add_session_nonce_to_journalist.py index ea512ed85f..e0cb1e8ec1 100644 --- a/securedrop/alembic/versions/60f41bb14d98_add_session_nonce_to_journalist.py +++ b/securedrop/alembic/versions/60f41bb14d98_add_session_nonce_to_journalist.py @@ -16,7 +16,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: conn = op.get_bind() conn.execute("PRAGMA legacy_alter_table=ON") # Save existing journalist table. @@ -70,7 +70,7 @@ def upgrade(): op.drop_table('journalists_tmp') -def downgrade(): +def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('journalists', schema=None) as batch_op: batch_op.drop_column('session_nonce') diff --git a/securedrop/alembic/versions/6db892e17271_add_reply_uuid.py b/securedrop/alembic/versions/6db892e17271_add_reply_uuid.py index 6194e288f3..abe77ba9ed 100644 --- a/securedrop/alembic/versions/6db892e17271_add_reply_uuid.py +++ b/securedrop/alembic/versions/6db892e17271_add_reply_uuid.py @@ -17,7 +17,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: conn = op.get_bind() conn.execute("PRAGMA legacy_alter_table=ON") # Schema migration @@ -67,7 +67,7 @@ def upgrade(): op.drop_table("replies_tmp") -def downgrade(): +def downgrade() -> None: with op.batch_alter_table("replies", schema=None) as batch_op: batch_op.drop_column("uuid") diff --git a/securedrop/alembic/versions/92fba0be98e9_added_organization_name_field_in_.py b/securedrop/alembic/versions/92fba0be98e9_added_organization_name_field_in_.py index 4b06b3c523..7574b346a8 100644 --- a/securedrop/alembic/versions/92fba0be98e9_added_organization_name_field_in_.py +++ b/securedrop/alembic/versions/92fba0be98e9_added_organization_name_field_in_.py @@ -16,7 +16,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('instance_config', schema=None) as batch_op: batch_op.add_column(sa.Column('organization_name', sa.String(length=255), nullable=True)) @@ -24,7 +24,7 @@ def upgrade(): # ### end Alembic commands ### -def downgrade(): +def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('instance_config', schema=None) as batch_op: batch_op.drop_column('organization_name') diff --git a/securedrop/alembic/versions/a9fe328b053a_migrations_for_0_14_0.py b/securedrop/alembic/versions/a9fe328b053a_migrations_for_0_14_0.py index 2c22906c52..1e635f57e7 100644 --- a/securedrop/alembic/versions/a9fe328b053a_migrations_for_0_14_0.py +++ b/securedrop/alembic/versions/a9fe328b053a_migrations_for_0_14_0.py @@ -16,13 +16,13 @@ depends_on = None -def upgrade(): +def upgrade() -> None: with op.batch_alter_table('journalists', schema=None) as batch_op: batch_op.add_column(sa.Column('first_name', sa.String(length=255), nullable=True)) batch_op.add_column(sa.Column('last_name', sa.String(length=255), nullable=True)) -def downgrade(): +def downgrade() -> None: with op.batch_alter_table('journalists', schema=None) as batch_op: batch_op.drop_column('last_name') batch_op.drop_column('first_name') diff --git a/securedrop/alembic/versions/b060f38c0c31_drop_source_flagged.py b/securedrop/alembic/versions/b060f38c0c31_drop_source_flagged.py index b591a04590..607c79f6c4 100644 --- a/securedrop/alembic/versions/b060f38c0c31_drop_source_flagged.py +++ b/securedrop/alembic/versions/b060f38c0c31_drop_source_flagged.py @@ -16,12 +16,12 @@ depends_on = None -def upgrade(): +def upgrade() -> None: with op.batch_alter_table("sources", schema=None) as batch_op: batch_op.drop_column("flagged") -def downgrade(): +def downgrade() -> None: # You might be tempted to try Alembic's batch_ops for the # downgrade too. Don't. SQLite's unnamed check constraints require # kludges. diff --git a/securedrop/alembic/versions/b58139cfdc8c_add_checksum_columns_revoke_table.py b/securedrop/alembic/versions/b58139cfdc8c_add_checksum_columns_revoke_table.py index ce00f6f53c..37ad5ae4d5 100644 --- a/securedrop/alembic/versions/b58139cfdc8c_add_checksum_columns_revoke_table.py +++ b/securedrop/alembic/versions/b58139cfdc8c_add_checksum_columns_revoke_table.py @@ -27,7 +27,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: with op.batch_alter_table("replies", schema=None) as batch_op: batch_op.add_column(sa.Column("checksum", sa.String(length=255), nullable=True)) @@ -88,7 +88,7 @@ def upgrade(): raise -def downgrade(): +def downgrade() -> None: op.drop_table("revoked_tokens") with op.batch_alter_table("submissions", schema=None) as batch_op: diff --git a/securedrop/alembic/versions/d9d36b6f4d1e_remove_partial_index_on_valid_until_set_.py b/securedrop/alembic/versions/d9d36b6f4d1e_remove_partial_index_on_valid_until_set_.py index d865a3621e..131f684c52 100644 --- a/securedrop/alembic/versions/d9d36b6f4d1e_remove_partial_index_on_valid_until_set_.py +++ b/securedrop/alembic/versions/d9d36b6f4d1e_remove_partial_index_on_valid_until_set_.py @@ -17,7 +17,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### # remove the old partial index on valid_until, as alembic can't handle it. op.execute(sa.text('DROP INDEX IF EXISTS ix_one_active_instance_config')) @@ -50,7 +50,7 @@ def upgrade(): # ### end Alembic commands ### -def downgrade(): +def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('instance_config', schema=None) as batch_op: batch_op.alter_column('valid_until', diff --git a/securedrop/alembic/versions/de00920916bf_updates_journalists_otp_secret_length_.py b/securedrop/alembic/versions/de00920916bf_updates_journalists_otp_secret_length_.py index 7a3bb635de..04727b330a 100644 --- a/securedrop/alembic/versions/de00920916bf_updates_journalists_otp_secret_length_.py +++ b/securedrop/alembic/versions/de00920916bf_updates_journalists_otp_secret_length_.py @@ -16,7 +16,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('journalists', schema=None) as batch_op: batch_op.alter_column('otp_secret', @@ -27,7 +27,7 @@ def upgrade(): # ### end Alembic commands ### -def downgrade(): +def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('journalists', schema=None) as batch_op: batch_op.alter_column('otp_secret', diff --git a/securedrop/alembic/versions/e0a525cbab83_add_column_to_track_source_deletion_of_.py b/securedrop/alembic/versions/e0a525cbab83_add_column_to_track_source_deletion_of_.py index b33e6fcd30..fc44fc8b26 100644 --- a/securedrop/alembic/versions/e0a525cbab83_add_column_to_track_source_deletion_of_.py +++ b/securedrop/alembic/versions/e0a525cbab83_add_column_to_track_source_deletion_of_.py @@ -16,7 +16,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: conn = op.get_bind() conn.execute("PRAGMA legacy_alter_table=ON") # Schema migration @@ -64,6 +64,6 @@ def upgrade(): op.drop_table("replies_tmp") -def downgrade(): +def downgrade() -> None: with op.batch_alter_table("replies", schema=None) as batch_op: batch_op.drop_column("deleted_by_source") diff --git a/securedrop/alembic/versions/f2833ac34bb6_add_uuid_column_for_users_table.py b/securedrop/alembic/versions/f2833ac34bb6_add_uuid_column_for_users_table.py index 7106632756..4c59055e3b 100644 --- a/securedrop/alembic/versions/f2833ac34bb6_add_uuid_column_for_users_table.py +++ b/securedrop/alembic/versions/f2833ac34bb6_add_uuid_column_for_users_table.py @@ -17,7 +17,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: conn = op.get_bind() conn.execute("PRAGMA legacy_alter_table=ON") # Save existing journalist table. @@ -72,6 +72,6 @@ def upgrade(): op.drop_table("journalists_tmp") -def downgrade(): +def downgrade() -> None: with op.batch_alter_table("journalists", schema=None) as batch_op: batch_op.drop_column("uuid") diff --git a/securedrop/alembic/versions/faac8092c123_enable_security_pragmas.py b/securedrop/alembic/versions/faac8092c123_enable_security_pragmas.py index cae82e3af4..13d030a875 100644 --- a/securedrop/alembic/versions/faac8092c123_enable_security_pragmas.py +++ b/securedrop/alembic/versions/faac8092c123_enable_security_pragmas.py @@ -16,11 +16,11 @@ depends_on = None -def upgrade(): +def upgrade() -> None: conn = op.get_bind() conn.execute(sa.text('PRAGMA secure_delete = ON')) conn.execute(sa.text('PRAGMA auto_vacuum = FULL')) -def downgrade(): +def downgrade() -> None: pass diff --git a/securedrop/alembic/versions/fccf57ceef02_create_submission_uuid_column.py b/securedrop/alembic/versions/fccf57ceef02_create_submission_uuid_column.py index 8fa8f1c486..a0b2b7214f 100644 --- a/securedrop/alembic/versions/fccf57ceef02_create_submission_uuid_column.py +++ b/securedrop/alembic/versions/fccf57ceef02_create_submission_uuid_column.py @@ -17,7 +17,7 @@ depends_on = None -def upgrade(): +def upgrade() -> None: conn = op.get_bind() conn.execute("PRAGMA legacy_alter_table=ON") # Schema migration @@ -64,6 +64,6 @@ def upgrade(): op.drop_table("submissions_tmp") -def downgrade(): +def downgrade() -> None: with op.batch_alter_table("submissions", schema=None) as batch_op: batch_op.drop_column("uuid") diff --git a/securedrop/bin/run-mypy b/securedrop/bin/run-mypy new file mode 100755 index 0000000000..771d0bbe79 --- /dev/null +++ b/securedrop/bin/run-mypy @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +# Run from repository root +REPOROOT=$(git rev-parse --show-toplevel) +cd "${REPOROOT}" + +if [ "$(command -v mypy)" ]; then + mypy ./securedrop ./admin --namespace-packages --explicit-package-bases "$@" +elif [ -d "/opt/venvs/securedrop-app-code/" ]; then + # Inside the dev container, but no mypy + echo "Could not find mypy" + exit 1 +else + ./securedrop/bin/dev-shell ./bin/run-mypy +fi diff --git a/securedrop/models.py b/securedrop/models.py index d0778fb38c..16c4f0d127 100644 --- a/securedrop/models.py +++ b/securedrop/models.py @@ -18,7 +18,7 @@ from jinja2 import Markup from passlib.hash import argon2 from sqlalchemy import ForeignKey -from sqlalchemy.orm import relationship, backref, Query, RelationshipProperty +from sqlalchemy.orm import relationship, backref, Query from sqlalchemy import Column, Integer, String, Boolean, DateTime, LargeBinary from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound @@ -76,7 +76,7 @@ class Source(db.Model): last_updated = Column(DateTime) star = relationship( "SourceStar", uselist=False, backref="source" - ) # type: RelationshipProperty[SourceStar] + ) # sources are "pending" and don't get displayed to journalists until they # submit something @@ -186,7 +186,7 @@ class Submission(db.Model): source = relationship( "Source", backref=backref("submissions", order_by=id, cascade="delete") - ) # type: RelationshipProperty[Source] + ) filename = Column(String(255), nullable=False) size = Column(Integer, nullable=False) @@ -266,13 +266,13 @@ class Reply(db.Model): backref=backref( 'replies', order_by=id) - ) # type: RelationshipProperty[Journalist] + ) source_id = Column(Integer, ForeignKey('sources.id')) source = relationship( "Source", backref=backref("replies", order_by=id, cascade="delete") - ) # type: RelationshipProperty[Source] + ) filename = Column(String(255), nullable=False) size = Column(Integer, nullable=False) @@ -414,33 +414,33 @@ class Journalist(db.Model): username = Column(String(255), nullable=False, unique=True) first_name = Column(String(255), nullable=True) last_name = Column(String(255), nullable=True) - pw_salt = Column(LargeBinary(32), nullable=True) # type: Column[Optional[bytes]] - pw_hash = Column(LargeBinary(256), nullable=True) # type: Column[Optional[bytes]] - is_admin = Column(Boolean) # type: Column[Optional[bool]] + pw_salt = Column(LargeBinary(32), nullable=True) + pw_hash = Column(LargeBinary(256), nullable=True) + is_admin = Column(Boolean) session_nonce = Column(Integer, nullable=False, default=0) otp_secret = Column(String(32), default=pyotp.random_base32) - is_totp = Column(Boolean, default=True) # type: Column[Optional[bool]] - hotp_counter = Column(Integer, default=0) # type: Column[Optional[int]] + is_totp = Column(Boolean, default=True) + hotp_counter = Column(Integer, default=0) last_token = Column(String(6)) created_on = Column( DateTime, default=datetime.datetime.utcnow - ) # type: Column[Optional[datetime.datetime]] - last_access = Column(DateTime) # type: Column[Optional[datetime.datetime]] - passphrase_hash = Column(String(256)) # type: Column[Optional[str]] + ) + last_access = Column(DateTime) + passphrase_hash = Column(String(256)) login_attempts = relationship( "JournalistLoginAttempt", backref="journalist", cascade="all, delete" - ) # type: RelationshipProperty[JournalistLoginAttempt] + ) revoked_tokens = relationship( "RevokedToken", backref="journalist", cascade="all, delete" - ) # type: RelationshipProperty[RevokedToken] + ) MIN_USERNAME_LEN = 3 MIN_NAME_LEN = 0 @@ -866,10 +866,10 @@ class SeenFile(db.Model): journalist_id = Column(Integer, ForeignKey("journalists.id"), nullable=False) file = relationship( "Submission", backref=backref("seen_files", lazy="dynamic", cascade="all,delete") - ) # type: RelationshipProperty[Submission] + ) journalist = relationship( "Journalist", backref=backref("seen_files") - ) # type: RelationshipProperty[Journalist] + ) class SeenMessage(db.Model): @@ -880,10 +880,10 @@ class SeenMessage(db.Model): journalist_id = Column(Integer, ForeignKey("journalists.id"), nullable=False) message = relationship( "Submission", backref=backref("seen_messages", lazy="dynamic", cascade="all,delete") - ) # type: RelationshipProperty[Submission] + ) journalist = relationship( "Journalist", backref=backref("seen_messages") - ) # type: RelationshipProperty[Journalist] + ) class SeenReply(db.Model): @@ -894,10 +894,10 @@ class SeenReply(db.Model): journalist_id = Column(Integer, ForeignKey("journalists.id"), nullable=False) reply = relationship( "Reply", backref=backref("seen_replies", cascade="all,delete") - ) # type: RelationshipProperty[Reply] + ) journalist = relationship( "Journalist", backref=backref("seen_replies") - ) # type: RelationshipProperty[Journalist] + ) class JournalistLoginAttempt(db.Model): @@ -910,7 +910,7 @@ class JournalistLoginAttempt(db.Model): timestamp = Column( DateTime, default=datetime.datetime.utcnow - ) # type: Column[Optional[datetime.datetime]] + ) journalist_id = Column(Integer, ForeignKey('journalists.id'), nullable=False) def __init__(self, journalist: Journalist) -> None: diff --git a/securedrop/requirements/python3/develop-requirements.in b/securedrop/requirements/python3/develop-requirements.in index 5d56062e37..7352f6676c 100644 --- a/securedrop/requirements/python3/develop-requirements.in +++ b/securedrop/requirements/python3/develop-requirements.in @@ -16,7 +16,6 @@ importlib-resources markupsafe>=2.0 molecule>=3.0.1 molecule-vagrant>=0.3 -mypy>=0.761 # Needed for ansible network filter # http://docs.ansible.com/ansible/latest/playbooks_filters_ipaddr.html netaddr @@ -39,4 +38,3 @@ setuptools>=56.0.0 testinfra>=5.3.1 urllib3>=1.26.5 yamllint -sqlalchemy-stubs==0.3 diff --git a/securedrop/requirements/python3/develop-requirements.txt b/securedrop/requirements/python3/develop-requirements.txt index 6bc89a3343..b43a24633a 100644 --- a/securedrop/requirements/python3/develop-requirements.txt +++ b/securedrop/requirements/python3/develop-requirements.txt @@ -462,28 +462,6 @@ more-itertools==7.2.0 \ --hash=sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832 \ --hash=sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4 # via zipp -mypy-extensions==0.4.3 \ - --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ - --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 - # via mypy -mypy==0.761 \ - --hash=sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a \ - --hash=sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7 \ - --hash=sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2 \ - --hash=sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474 \ - --hash=sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0 \ - --hash=sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217 \ - --hash=sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749 \ - --hash=sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6 \ - --hash=sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf \ - --hash=sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36 \ - --hash=sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b \ - --hash=sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72 \ - --hash=sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1 \ - --hash=sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1 - # via - # -r requirements/python3/develop-requirements.in - # sqlalchemy-stubs netaddr==0.7.19 \ --hash=sha256:38aeec7cdd035081d3a4c306394b19d677623bf76fa0913f6695127c7753aefd \ --hash=sha256:56b3558bd71f3f6999e4c52e349f38660e54a7a8a9943335f73dfc96883e08ca @@ -775,10 +753,6 @@ smmap2==2.0.3 \ --hash=sha256:b78ee0f1f5772d69ff50b1cbdb01b8c6647a8354f02f23b488cf4b2cfc923956 \ --hash=sha256:c7530db63f15f09f8251094b22091298e82bf6c699a6b8344aaaef3f2e1276c3 # via gitdb2 -sqlalchemy-stubs==0.3 \ - --hash=sha256:a3318c810697164e8c818aa2d90bac570c1a0e752ced3ec25455b309c0bee8fd \ - --hash=sha256:ca1250605a39648cc433f5c70cb1a6f9fe0b60bdda4c51e1f9a2ab3651daadc8 - # via -r requirements/python3/develop-requirements.in stevedore==1.28.0 \ --hash=sha256:e3d96b2c4e882ec0c1ff95eaebf7b575a779fd0ccb4c741b9832bed410d58b3d \ --hash=sha256:f1c7518e7b160336040fee272174f1f7b29a46febb3632502a8f2055f973d60b @@ -806,36 +780,6 @@ tree-format==0.1.2 \ --hash=sha256:a538523aa78ae7a4b10003b04f3e1b37708e0e089d99c9d3b9e1c71384c9a7f9 \ --hash=sha256:b5056228dbedde1fb81b79f71fb0c23c98e9d365230df9b29af76e8d8003de11 # via molecule -typed-ast==1.4.1 \ - --hash=sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355 \ - --hash=sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919 \ - --hash=sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa \ - --hash=sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652 \ - --hash=sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75 \ - --hash=sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01 \ - --hash=sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d \ - --hash=sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1 \ - --hash=sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907 \ - --hash=sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c \ - --hash=sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3 \ - --hash=sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b \ - --hash=sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614 \ - --hash=sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb \ - --hash=sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b \ - --hash=sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41 \ - --hash=sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6 \ - --hash=sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34 \ - --hash=sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe \ - --hash=sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4 \ - --hash=sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7 - # via mypy -typing-extensions==3.7.4.1 \ - --hash=sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2 \ - --hash=sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d \ - --hash=sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575 - # via - # mypy - # sqlalchemy-stubs urllib3==1.26.6 \ --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \ --hash=sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f diff --git a/securedrop/requirements/python3/test-requirements.in b/securedrop/requirements/python3/test-requirements.in index e5462861ce..92ab623aaf 100644 --- a/securedrop/requirements/python3/test-requirements.in +++ b/securedrop/requirements/python3/test-requirements.in @@ -20,3 +20,11 @@ selenium>=3.141.0 tbselenium>=0.5.2 pyvirtualdisplay urllib3>=1.26.5 +# mypy and co. +mypy>=0.761 +sqlalchemy-stubs==0.4 +types-mock +types-PyYAML +types-redis +types-requests +types-setuptools diff --git a/securedrop/requirements/python3/test-requirements.txt b/securedrop/requirements/python3/test-requirements.txt index 177e0e1b4d..807692c162 100644 --- a/securedrop/requirements/python3/test-requirements.txt +++ b/securedrop/requirements/python3/test-requirements.txt @@ -103,6 +103,37 @@ mock==2.0.0 \ --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba # via -r requirements/python3/test-requirements.in +mypy-extensions==0.4.3 \ + --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ + --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 + # via mypy +mypy==0.942 \ + --hash=sha256:0e2dd88410937423fba18e57147dd07cd8381291b93d5b1984626f173a26543e \ + --hash=sha256:10daab80bc40f84e3f087d896cdb53dc811a9f04eae4b3f95779c26edee89d16 \ + --hash=sha256:17e44649fec92e9f82102b48a3bf7b4a5510ad0cd22fa21a104826b5db4903e2 \ + --hash=sha256:1a0459c333f00e6a11cbf6b468b870c2b99a906cb72d6eadf3d1d95d38c9352c \ + --hash=sha256:246e1aa127d5b78488a4a0594bd95f6d6fb9d63cf08a66dafbff8595d8891f67 \ + --hash=sha256:2b184db8c618c43c3a31b32ff00cd28195d39e9c24e7c3b401f3db7f6e5767f5 \ + --hash=sha256:2bc249409a7168d37c658e062e1ab5173300984a2dada2589638568ddc1db02b \ + --hash=sha256:3841b5433ff936bff2f4dc8d54cf2cdbfea5d8e88cedfac45c161368e5770ba6 \ + --hash=sha256:4c3e497588afccfa4334a9986b56f703e75793133c4be3a02d06a3df16b67a58 \ + --hash=sha256:5bf44840fb43ac4074636fd47ee476d73f0039f4f54e86d7265077dc199be24d \ + --hash=sha256:64235137edc16bee6f095aba73be5334677d6f6bdb7fa03cfab90164fa294a17 \ + --hash=sha256:6776e5fa22381cc761df53e7496a805801c1a751b27b99a9ff2f0ca848c7eca0 \ + --hash=sha256:6ce34a118d1a898f47def970a2042b8af6bdcc01546454726c7dd2171aa6dfca \ + --hash=sha256:6f6ad963172152e112b87cc7ec103ba0f2db2f1cd8997237827c052a3903eaa6 \ + --hash=sha256:6f7106cbf9cc2f403693bf50ed7c9fa5bb3dfa9007b240db3c910929abe2a322 \ + --hash=sha256:7742d2c4e46bb5017b51c810283a6a389296cda03df805a4f7869a6f41246534 \ + --hash=sha256:9521c1265ccaaa1791d2c13582f06facf815f426cd8b07c3a485f486a8ffc1f3 \ + --hash=sha256:a1b383fe99678d7402754fe90448d4037f9512ce70c21f8aee3b8bf48ffc51db \ + --hash=sha256:b840cfe89c4ab6386c40300689cd8645fc8d2d5f20101c7f8bd23d15fca14904 \ + --hash=sha256:d8d3ba77e56b84cd47a8ee45b62c84b6d80d32383928fe2548c9a124ea0a725c \ + --hash=sha256:dcd955f36e0180258a96f880348fbca54ce092b40fbb4b37372ae3b25a0b0a46 \ + --hash=sha256:e865fec858d75b78b4d63266c9aff770ecb6a39dfb6d6b56c47f7f8aba6baba8 \ + --hash=sha256:edf7237137a1a9330046dbb14796963d734dd740a98d5e144a3eb1d267f5f9ee + # via + # -r requirements/python3/test-requirements.in + # sqlalchemy-stubs packaging==20.4 \ --hash=sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8 \ --hash=sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181 @@ -256,6 +287,10 @@ six==1.11.0 \ # mock # packaging # pathlib2 +sqlalchemy-stubs==0.4 \ + --hash=sha256:5eec7aa110adf9b957b631799a72fef396b23ff99fe296df726645d01e312aa5 \ + --hash=sha256:c665d6dd4482ef642f01027fa06c3d5e91befabb219dc71fc2a09e7d7695f7ae + # via -r requirements/python3/test-requirements.in tbselenium==0.5.2 \ --hash=sha256:84dcc3f250b0c6bb4ce4bdb0d62de1d086343334019accb5469c21897afebb06 # via -r requirements/python3/test-requirements.in @@ -265,6 +300,40 @@ toml==0.10.1 \ # via # pep517 # pytest +tomli==2.0.1 \ + --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ + --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f + # via mypy +types-mock==4.0.12 \ + --hash=sha256:81f98e66bddde1b2a8d96c15c084d5d3ed96cf6137bca9ba8aabc6b6865b9ca7 \ + --hash=sha256:f8b1dbe128dc2d5aa038876a8b497dfd1a9405210a5eb3dc4b4757c2e254627c + # via -r requirements/python3/test-requirements.in +types-pyyaml==6.0.5 \ + --hash=sha256:2fd21310870addfd51db621ad9f3b373f33ee3cbb81681d70ef578760bd22d35 \ + --hash=sha256:464e050914f3d1d83a8c038e1cf46da5cb96b7cd02eaa096bcaa03675edd8a2e + # via -r requirements/python3/test-requirements.in +types-redis==4.1.18 \ + --hash=sha256:93bc85db6fb4634e85eff8b64cb662d47a55141b532085f4a99c70b174e65e8d \ + --hash=sha256:b00fc56074d8ef0d7f52f5dc3ebfa2cc2f1970b384c8c83733ebab940b1f985e + # via -r requirements/python3/test-requirements.in +types-requests==2.27.15 \ + --hash=sha256:2d371183c535208d2cc8fe7473d9b49c344c7077eb70302eb708638fb86086a8 \ + --hash=sha256:77d09182a68e447e9e8b0ffc21abf54618b96f07689dffbb6a41cf0356542969 + # via -r requirements/python3/test-requirements.in +types-setuptools==57.4.11 \ + --hash=sha256:262f7406e0c7d705ad6bb4526b5b761fa500bf99eab74de85ac3592187d62935 \ + --hash=sha256:fbb0647569d6fb2f6bc472402265250c0ffa53e60180d2cbfee9e84f085921f0 + # via -r requirements/python3/test-requirements.in +types-urllib3==1.26.11 \ + --hash=sha256:24d64e441168851eb05f1d022de18ae31558f5649c8f1117e384c2e85e31315b \ + --hash=sha256:bd0abc01e9fb963e4fddd561a56d21cc371b988d1245662195c90379077139cd + # via types-requests +typing-extensions==4.1.1 \ + --hash=sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42 \ + --hash=sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2 + # via + # mypy + # sqlalchemy-stubs urllib3==1.26.6 \ --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \ --hash=sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f diff --git a/securedrop/secure_tempfile.py b/securedrop/secure_tempfile.py index 4e6f1cccc9..631667d4be 100644 --- a/securedrop/secure_tempfile.py +++ b/securedrop/secure_tempfile.py @@ -78,7 +78,7 @@ def initialize_cipher(self) -> None: self.encryptor = self.cipher.encryptor() self.decryptor = self.cipher.decryptor() - def write(self, data: Union[bytes, str]) -> None: + def write(self, data: Union[bytes, str]) -> int: """Write `data` to the secure temporary file. This method may be called any number of times following instance initialization, but after calling :meth:`read`, you cannot write to the file @@ -93,7 +93,7 @@ def write(self, data: Union[bytes, str]) -> None: else: data_as_bytes = data - self.file.write(self.encryptor.update(data_as_bytes)) + return self.file.write(self.encryptor.update(data_as_bytes)) def read(self, count: Optional[int] = None) -> bytes: """Read `data` from the secure temporary file. This method may