Added counter field and updated confirmation_id (#473)

Closes #472 
Closes #460 

Removed microseconds from timestamp
Added counter field to Submission table/DAO/DTO
Added alembic script to add field, add unique constraint for filing id
and counter, and set existing submission counters
Updated add_submission to get the latest count and add 1, or start from
1 if new for the filing period
Updated pytests to test counter

db_revisions/versions/6ec12afa5b37_add_counter_to_submission_table.py

@@ -0,0 +1,52 @@
+"""add counter to submission table
+
+Revision ID: 6ec12afa5b37
+Revises: 26f11ac15b3c
+Create Date: 2024-10-28 10:52:22.353469
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "6ec12afa5b37"
+down_revision: Union[str, None] = "63138f5cf036"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # create the new column as nullable so it doesn't error with existing data
+    with op.batch_alter_table("submission", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("counter", sa.Integer, nullable=True))
+        batch_op.create_unique_constraint("unique_filing_counter", ["filing", "counter"])
+
+    # run a counter of each submission in a given filing id, ordered by the submission id,
+    # which is the PK incrementor.  This will give us accurate counts of 1, 2, 3, ... for
+    # each submission per filing id.
+    conn = op.get_bind()
+    conn.execute(
+        sa.text(
+            """
+        WITH counts AS (
+            SELECT id, filing, ROW_NUMBER() OVER (PARTITION BY filing ORDER BY id) AS row_num FROM submission
+        )
+        UPDATE submission SET counter = counts.row_num
+        FROM counts
+        WHERE submission.id = counts.id
+        """
+        )
+    )
+
+    # set the counter column to required now that existing data is set.
+    with op.batch_alter_table("submission", schema=None) as batch_op:
+        batch_op.alter_column("counter", nullable=False)
+
+
+def downgrade() -> None:
+    op.drop_constraint(constraint_name="unique_filing_counter", table_name="submission")
+    op.drop_column("submission", "counter")

src/sbl_filing_api/entities/models/dao.py

@@ -2,7 +2,7 @@
 from datetime import datetime
 from typing import Any, List
 from sqlalchemy import Enum as SAEnum, String
-from sqlalchemy import ForeignKey, func
+from sqlalchemy import ForeignKey, func, UniqueConstraint
 from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase, relationship
 from sqlalchemy.ext.asyncio import AsyncAttrs
 from sqlalchemy.types import JSON
@@ -26,6 +26,7 @@ class SubmissionDAO(Base):
     __tablename__ = "submission"
     id: Mapped[int] = mapped_column(index=True, primary_key=True, autoincrement=True)
     filing: Mapped[int] = mapped_column(ForeignKey("filing.id"))
+    counter: Mapped[int]
     submitter_id: Mapped[int] = mapped_column(ForeignKey("user_action.id"))
     submitter: Mapped[UserActionDAO] = relationship(lazy="selectin", foreign_keys=[submitter_id])
     accepter_id: Mapped[int] = mapped_column(ForeignKey("user_action.id"), nullable=True)
@@ -37,8 +38,10 @@ class SubmissionDAO(Base):
     filename: Mapped[str]
     total_records: Mapped[int] = mapped_column(nullable=True)
 
+    __table_args__ = (UniqueConstraint("filing", "counter", name="unique_filing_counter"),)
+
     def __str__(self):
-        return f"Submission ID: {self.id}, State: {self.state}, Ruleset: {self.validation_ruleset_version}, Filing Period: {self.filing}, Submission: {self.submission_time}"
+        return f"Submission ID: {self.id}, Counter: {self.counter}, State: {self.state}, Ruleset: {self.validation_ruleset_version}, Filing Period: {self.filing}, Submission: {self.submission_time}"
 
 
 class FilingPeriodDAO(Base):

src/sbl_filing_api/entities/models/dto.py

@@ -18,6 +18,7 @@ class SubmissionDTO(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
     id: int | None = None
+    counter: int
     state: SubmissionState | None = None
     validation_ruleset_version: str | None = None
     validation_results: Dict[str, Any] | None = None
@@ -27,6 +28,13 @@ class SubmissionDTO(BaseModel):
     submitter: UserActionDTO
     accepter: UserActionDTO | None = None
 
+    @model_validator(mode="after")
+    def validate_fi(self) -> "SubmissionDTO":
+        print(f"Self: {self}")
+        self.id = self.counter
+        print(f"Self: {self}")
+        return self
+
 
 class FilingTaskDTO(BaseModel):
     model_config = ConfigDict(from_attributes=True)

src/sbl_filing_api/entities/repos/submission_repo.py

@@ -56,6 +56,12 @@ async def get_submission(session: AsyncSession, submission_id: int) -> Submissio
     return result[0] if result else None
 
 
+async def get_submission_by_counter(session: AsyncSession, lei: str, filing_period: str, counter: int) -> SubmissionDAO:
+    filing = await get_filing(session, lei=lei, filing_period=filing_period)
+    result = await query_helper(session, SubmissionDAO, filing=filing.id, counter=counter)
+    return result[0] if result else None
+
+
 async def get_filing(session: AsyncSession, lei: str, filing_period: str) -> FilingDAO:
     result = await query_helper(session, FilingDAO, lei=lei, filing_period=filing_period)
     if result:
@@ -98,8 +104,15 @@ async def get_user_actions(session: AsyncSession) -> List[UserActionDAO]:
 
 
 async def add_submission(session: AsyncSession, filing_id: int, filename: str, submitter_id: int) -> SubmissionDAO:
+    stmt = select(SubmissionDAO).filter_by(filing=filing_id).order_by(desc(SubmissionDAO.counter)).limit(1)
+    last_sub = await session.scalar(stmt)
+    current_count = last_sub.counter if last_sub else 0
     new_sub = SubmissionDAO(
-        filing=filing_id, state=SubmissionState.SUBMISSION_STARTED, filename=filename, submitter_id=submitter_id
+        filing=filing_id,
+        state=SubmissionState.SUBMISSION_STARTED,
+        filename=filename,
+        submitter_id=submitter_id,
+        counter=(current_count + 1),
     )
     # this returns the attached object, most importantly with the new submission id
     new_sub = await session.merge(new_sub)

src/sbl_filing_api/routers/filing.py

@@ -159,7 +159,9 @@ async def sign_filing(request: Request, lei: str, period_code: str):
             action_type=UserActionType.SIGN,
         ),
     )
-    filing.confirmation_id = lei + "-" + period_code + "-" + str(latest_sub.id) + "-" + str(sig.timestamp.timestamp())
+    filing.confirmation_id = (
+        lei + "-" + period_code + "-" + str(latest_sub.counter) + "-" + str(int(sig.timestamp.timestamp()))
+    )
     filing.signatures.append(sig)
     return await repo.upsert_filing(request.state.db_session, filing)
 
@@ -191,7 +193,7 @@ async def upload_file(request: Request, lei: str, period_code: str, file: Upload
         submission = await repo.add_submission(request.state.db_session, filing.id, file.filename, submitter.id)
         try:
             submission_processor.upload_to_storage(
-                period_code, lei, submission.id, content, file.filename.split(".")[-1]
+                period_code, lei, submission.counter, content, file.filename.split(".")[-1]
             )
 
             submission.state = SubmissionState.SUBMISSION_UPLOADED
@@ -259,24 +261,24 @@ async def get_submission_latest(request: Request, lei: str, period_code: str):
     return Response(status_code=status.HTTP_204_NO_CONTENT)
 
 
-@router.get("/institutions/{lei}/filings/{period_code}/submissions/{id}", response_model=SubmissionDTO | None)
+@router.get("/institutions/{lei}/filings/{period_code}/submissions/{counter}", response_model=SubmissionDTO | None)
 @requires("authenticated")
-async def get_submission(request: Request, response: Response, id: int):
-    result = await repo.get_submission(request.state.db_session, id)
+async def get_submission(request: Request, response: Response, counter: int, lei: str, period_code: str):
+    result = await repo.get_submission_by_counter(request.state.db_session, lei, period_code, counter)
     if result:
         return result
     response.status_code = status.HTTP_404_NOT_FOUND
 
 
-@router.put("/institutions/{lei}/filings/{period_code}/submissions/{id}/accept", response_model=SubmissionDTO)
+@router.put("/institutions/{lei}/filings/{period_code}/submissions/{counter}/accept", response_model=SubmissionDTO)
 @requires("authenticated")
-async def accept_submission(request: Request, id: int, lei: str, period_code: str):
-    submission = await repo.get_submission(request.state.db_session, id)
+async def accept_submission(request: Request, counter: int, lei: str, period_code: str):
+    submission = await repo.get_submission_by_counter(request.state.db_session, lei, period_code, counter)
     if not submission:
         raise RegTechHttpException(
             status_code=status.HTTP_404_NOT_FOUND,
             name="Submission Not Found",
-            detail=f"Submission ID {id} does not exist, cannot accept a non-existing submission.",
+            detail=f"Submission {counter} for LEI {lei} in filing period {period_code} does not exist, cannot accept a non-existing submission.",
         )
     if (
         submission.state != SubmissionState.VALIDATION_SUCCESSFUL
@@ -285,7 +287,7 @@ async def accept_submission(request: Request, id: int, lei: str, period_code: st
         raise RegTechHttpException(
             status_code=status.HTTP_403_FORBIDDEN,
             name="Submission Action Forbidden",
-            detail=f"Submission {id} for LEI {lei} in filing period {period_code} is not in an acceptable state.  Submissions must be validated successfully or with only warnings to be accepted.",
+            detail=f"Submission {counter} for LEI {lei} in filing period {period_code} is not in an acceptable state.  Submissions must be validated successfully or with only warnings to be accepted.",
         )
 
     accepter = await repo.add_user_action(
@@ -367,13 +369,13 @@ async def get_latest_submission_report(request: Request, lei: str, period_code:
         SubmissionState.SUBMISSION_ACCEPTED,
     ]:
         file_data = submission_processor.get_from_storage(
-            period_code, lei, str(latest_sub.id) + submission_processor.REPORT_QUALIFIER
+            period_code, lei, str(latest_sub.counter) + submission_processor.REPORT_QUALIFIER
         )
         return StreamingResponse(
             content=file_data,
             media_type="text/csv",
             headers={
-                "Content-Disposition": f'attachment; filename="{latest_sub.id}_validation_report.csv"',
+                "Content-Disposition": f'attachment; filename="{latest_sub.counter}_validation_report.csv"',
                 "Cache-Control": "no-store",
             },
         )
@@ -386,34 +388,34 @@ async def get_latest_submission_report(request: Request, lei: str, period_code:
 
 
 @router.get(
-    "/institutions/{lei}/filings/{period_code}/submissions/{id}/report",
+    "/institutions/{lei}/filings/{period_code}/submissions/{counter}/report",
     responses={200: {"content": {"text/plain; charset=utf-8": {}}}},
 )
 @requires("authenticated")
-async def get_submission_report(request: Request, response: Response, lei: str, period_code: str, id: int):
-    sub = await repo.get_submission(request.state.db_session, id)
+async def get_submission_report(request: Request, response: Response, lei: str, period_code: str, counter: int):
+    sub = await repo.get_submission_by_counter(request.state.db_session, lei, period_code, counter)
     if sub and sub.state in [
         SubmissionState.VALIDATION_SUCCESSFUL,
         SubmissionState.VALIDATION_WITH_ERRORS,
         SubmissionState.VALIDATION_WITH_WARNINGS,
         SubmissionState.SUBMISSION_ACCEPTED,
     ]:
         file_data = submission_processor.get_from_storage(
-            period_code, lei, str(sub.id) + submission_processor.REPORT_QUALIFIER
+            period_code, lei, str(sub.counter) + submission_processor.REPORT_QUALIFIER
         )
         return StreamingResponse(
             content=file_data,
             media_type="text/csv",
             headers={
-                "Content-Disposition": f'attachment; filename="{sub.id}_validation_report.csv"',
+                "Content-Disposition": f'attachment; filename="{sub.counter}_validation_report.csv"',
                 "Cache-Control": "no-store",
             },
         )
     else:
         raise RegTechHttpException(
             status_code=status.HTTP_404_NOT_FOUND,
             name="Report Not Found",
-            detail=f"Report for ({id}) does not exist.",
+            detail=f"Report for ({lei}-{period_code}-{counter}) does not exist.",
         )
 
 

src/sbl_filing_api/services/submission_processor.py

@@ -75,7 +75,7 @@ async def validate_and_update_submission(
             submission.state = SubmissionState.VALIDATION_IN_PROGRESS
             submission = await update_submission(session, submission)
 
-            file_path = generate_file_path(period_code, lei, submission.id)
+            file_path = generate_file_path(period_code, lei, submission.counter)
 
             final_phase = ValidationPhase.LOGICAL
             all_findings = []
@@ -112,8 +112,9 @@ async def validate_and_update_submission(
                 error_count=sum([r.error_counts.total_count for r in all_findings]),
                 max_errors=settings.max_validation_errors,
             )
+            upload_to_storage(period_code, lei, str(submission.counter) + REPORT_QUALIFIER, submission_report)
 
-            upload_to_storage(period_code, lei, str(submission.id) + REPORT_QUALIFIER, submission_report)
+            upload_to_storage(period_code, lei, str(submission.counter) + REPORT_QUALIFIER, submission_report)
 
             if not exec_check["continue"]:
                 log.warning(f"Submission {submission.id} is expired, will not be updating final state with results.")
@@ -126,7 +127,8 @@ async def validate_and_update_submission(
             submission.state = SubmissionState.SUBMISSION_UPLOAD_MALFORMED
             await update_submission(session, submission)
 
-        except Exception:
+        except Exception as e:
+            print(f"{e}")
             log.exception("Validation for submission %d did not complete due to an unexpected error.", submission.id)
             submission.state = SubmissionState.VALIDATION_ERROR
             await update_submission(session, submission)