170 add validation error if an unknown exception happens during validation

db_revisions/versions/c89920be2e66_add_validation_error_state.py

@@ -0,0 +1,50 @@
+"""Add VALIDATION_ERROR state
+
+Revision ID: c89920be2e66
+Revises: 3f7e610035a6
+Create Date: 2024-04-18 13:06:48.162639
+
+"""
+from typing import Sequence, Union
+
+from alembic import op, context
+
+
+# revision identifiers, used by Alembic.
+revision: str = "c89920be2e66"
+down_revision: Union[str, None] = "3f7e610035a6"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+# fmt: off
+old_options = (
+    'SUBMISSION_ACCEPTED',
+    'SUBMISSION_STARTED',
+    'SUBMISSION_UPLOADED',
+    'SUBMISSION_UPLOAD_MALFORMED',
+    'UPLOAD_FAILED',
+    'VALIDATION_EXPIRED',
+    'VALIDATION_IN_PROGRESS',
+    'VALIDATION_WITH_ERRORS',
+    'VALIDATION_WITH_WARNINGS',
+    'VALIDATION_SUCCESSFUL',
+)
+new_options = tuple(sorted(old_options + ('VALIDATION_ERROR',)))
+# fmt: on
+
+
+def upgrade() -> None:
+    if "sqlite" not in context.get_context().dialect.name:
+        op.execute("ALTER TYPE submissionstate RENAME TO submissionstate_old")
+        op.execute(f"CREATE TYPE submissionstate AS ENUM{new_options}")
+        op.execute("ALTER TABLE submission ALTER COLUMN state TYPE submissionstate USING state::text::submissionstate")
+        op.execute("DROP TYPE submissionstate_old")
+
+
+def downgrade() -> None:
+    if "sqlite" not in context.get_context().dialect.name:
+        op.execute("ALTER TYPE submissionstate RENAME TO submissionstate_old")
+        op.execute(f"CREATE TYPE submissionstate AS ENUM{old_options}")
+        op.execute("ALTER TABLE submission ALTER COLUMN state TYPE submissionstate USING state::text::submissionstate")
+        op.execute("DROP TYPE submissionstate_old")

src/sbl_filing_api/entities/models/model_enums.py

@@ -14,6 +14,7 @@ class SubmissionState(str, Enum):
     SUBMISSION_UPLOAD_MALFORMED = "SUBMISSION_UPLOAD_MALFORMED"
     SUBMISSION_UPLOADED = "SUBMISSION_UPLOADED"
     UPLOAD_FAILED = "UPLOAD_FAILED"
+    VALIDATION_ERROR = "VALIDATION_ERROR"
     VALIDATION_EXPIRED = "VALIDATION_EXPIRED"
     VALIDATION_IN_PROGRESS = "VALIDATION_IN_PROGRESS"
     VALIDATION_SUCCESSFUL = "VALIDATION_SUCCESSFUL"

src/sbl_filing_api/services/submission_processor.py

@@ -35,7 +35,7 @@ async def validation_monitor(period_code: str, lei: str, submission: SubmissionD
             stack_info=True,
         )
         submission.state = SubmissionState.VALIDATION_EXPIRED
-        update_submission(submission)
+        await update_submission(submission)
 
 
 def validate_file_processable(file: UploadFile) -> None:
@@ -112,3 +112,13 @@ async def validate_and_update_submission(period_code: str, lei: str, submission:
         log.error("The file is malformed", re, exc_info=True, stack_info=True)
         submission.state = SubmissionState.SUBMISSION_UPLOAD_MALFORMED
         await update_submission(submission)
+
+    except Exception as e:
+        log.error(
+            f"Validation for submission {submission.id} did not complete due to an unexpected error.",
+            e,
+            exc_info=True,
+            stack_info=True,
+        )
+        submission.state = SubmissionState.VALIDATION_ERROR
+        await update_submission(submission)

tests/migrations/test_migrations.py

@@ -293,14 +293,6 @@ def test_migration_to_2e81179924b5(alembic_runner: MigrationContext, alembic_eng
     assert "hq_address_street_4" in [c["name"] for c in inspector.get_columns("contact_info")]
 
 
-def test_migration_to_4cd30d188352(alembic_runner: MigrationContext, alembic_engine: Engine):
-    alembic_runner.migrate_up_to("4cd30d188352")
-
-
-def test_migration_to_0040045eae14(alembic_runner: MigrationContext, alembic_engine: Engine):
-    alembic_runner.migrate_up_to("0040045eae14")
-
-
 def test_migration_to_102fb94a24cc(alembic_runner: MigrationContext, alembic_engine: Engine):
     alembic_runner.migrate_up_to("102fb94a24cc")
 

tests/services/test_submission_processor.py

@@ -217,6 +217,44 @@ async def test_validate_and_update_submission_malformed(
         assert mock_update_submission.mock_calls[0].args[0].state == SubmissionState.VALIDATION_IN_PROGRESS
         assert mock_update_submission.mock_calls[1].args[0].state == SubmissionState.SUBMISSION_UPLOAD_MALFORMED
 
+    async def test_validate_exception(
+        self,
+        mocker: MockerFixture,
+    ):
+        log_mock = mocker.patch("sbl_filing_api.services.submission_processor.log")
+
+        mock_sub = SubmissionDAO(
+            id=1,
+            filing=1,
+            state=SubmissionState.SUBMISSION_UPLOADED,
+            filename="submission.csv",
+        )
+
+        mock_update_submission = mocker.patch("sbl_filing_api.services.submission_processor.update_submission")
+        mock_update_submission.return_value = SubmissionDAO(
+            id=1,
+            filing=1,
+            state=SubmissionState.VALIDATION_IN_PROGRESS,
+            filename="submission.csv",
+        )
+
+        mocker.patch("pandas.read_csv")
+
+        mock_validation = mocker.patch("sbl_filing_api.services.submission_processor.validate_phases")
+        mock_validation.side_effect = KeyError(
+            "None of ['validation_id', 'record_no', 'field_name'] are in the columns"
+        )
+
+        await submission_processor.validate_and_update_submission("2024", "123456790", mock_sub, b"\x00\x00")
+
+        assert mock_update_submission.mock_calls[1].args[0].state == SubmissionState.VALIDATION_ERROR
+        assert log_mock.mock_calls[0].error.assert_called_with(
+            "Validation for submission 1 did not complete due to an unexpected error.",
+            mock_validation.side_effect,
+            exc_info=True,
+            stack_info=True,
+        )
+
     @pytest.mark.asyncio
     async def test_validation_monitor(
         self,