Merge branch 'main' into 404_user_actions_bug

Dockerfile

@@ -1,9 +1,9 @@
-FROM ghcr.io/cfpb/regtech/sbl/python-alpine:3.12
-
+FROM --platform=amd64 ghcr.io/cfpb/regtech/sbl/python-ubi8:3.12
 ENV UVICORN_LOG_LEVEL=info
 
 WORKDIR /usr/app
 RUN mkdir upload
+
 RUN pip install poetry
 
 COPY --chown=sbl:sbl poetry.lock pyproject.toml alembic.ini README.md ./

db_revisions/versions/6ec12afa5b37_add_counter_to_submission_table.py

@@ -0,0 +1,52 @@
+"""add counter to submission table
+
+Revision ID: 6ec12afa5b37
+Revises: 26f11ac15b3c
+Create Date: 2024-10-28 10:52:22.353469
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "6ec12afa5b37"
+down_revision: Union[str, None] = "63138f5cf036"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # create the new column as nullable so it doesn't error with existing data
+    with op.batch_alter_table("submission", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("counter", sa.Integer, nullable=True))
+        batch_op.create_unique_constraint("unique_filing_counter", ["filing", "counter"])
+
+    # run a counter of each submission in a given filing id, ordered by the submission id,
+    # which is the PK incrementor.  This will give us accurate counts of 1, 2, 3, ... for
+    # each submission per filing id.
+    conn = op.get_bind()
+    conn.execute(
+        sa.text(
+            """
+        WITH counts AS (
+            SELECT id, filing, ROW_NUMBER() OVER (PARTITION BY filing ORDER BY id) AS row_num FROM submission
+        )
+        UPDATE submission SET counter = counts.row_num
+        FROM counts
+        WHERE submission.id = counts.id
+        """
+        )
+    )
+
+    # set the counter column to required now that existing data is set.
+    with op.batch_alter_table("submission", schema=None) as batch_op:
+        batch_op.alter_column("counter", nullable=False)
+
+
+def downgrade() -> None:
+    op.drop_constraint(constraint_name="unique_filing_counter", table_name="submission")
+    op.drop_column("submission", "counter")

pyproject.toml

@@ -10,13 +10,13 @@ packages = [{ include = "sbl_filing_api", from = "src" }]
 python = ">=3.12,<4"
 sqlalchemy = "^2.0.35"
 psycopg2-binary = "^2.9.9"
-asyncpg = "^0.29.0"
+asyncpg = "^0.30.0"
 regtech-api-commons = {git = "https://github.com/cfpb/regtech-api-commons.git"}
 regtech-data-validator = {git = "https://github.com/cfpb/regtech-data-validator.git"}
 regtech-regex = {git = "https://github.com/cfpb/regtech-regex.git"}
+boto3 = "~1.34.0"
 python-multipart = "^0.0.12"
-boto3 = "^1.35.25"
-alembic = "^1.13.3"
+alembic = "^1.14.0"
 async-lru = "^2.0.4"
 ujson = "^5.10.0"
 
@@ -37,7 +37,7 @@ black = "24.8.0"
 
 
 [tool.poetry.group.load-testing.dependencies]
-locust = "^2.31.6"
+locust = "^2.32.2"
 httpx = "^0.27.2"
 
 

src/log-config.yml

@@ -20,30 +20,30 @@ handlers:
     class: logging.StreamHandler
     stream: ext://sys.stdout
 loggers:
-  uvicorn:
+  regtech_data_validator:
     level: INFO
     handlers:
       - default
-    propagate: no
-  uvicorn.error:
-    level: INFO
-  uvicorn.access:
+    propagate: false
+  regtech_api_commons:
     level: INFO
     handlers:
-      - access
-    propagate: no
-  regtech_data_validator:
+      - default
+    propagate: false
+  sbl_filing_api:
     level: INFO
     handlers:
       - default
     propagate: false
-  regtech_api_commons:
+  uvicorn:
     level: INFO
     handlers:
       - default
     propagate: false
-  sbl_filing_api:
+  uvicorn.error:
+    level: INFO
+  uvicorn.access:
     level: INFO
     handlers:
-      - default
+      - access
     propagate: false

src/sbl_filing_api/config.py

@@ -49,6 +49,7 @@ class Settings(BaseSettings):
     conn: PostgresDsn | None = None
 
     fs_upload_config: FsUploadConfig
+
     server_config: ServerConfig = ServerConfig()
 
     submission_file_type: str = "text/csv"
@@ -61,7 +62,7 @@ class Settings(BaseSettings):
 
     max_validation_errors: int = 1000000
     max_json_records: int = 10000
-    max_json_group_size: int = 0
+    max_json_group_size: int = 200
 
     def __init__(self, **data):
         super().__init__(**data)

src/sbl_filing_api/entities/models/dao.py

@@ -2,7 +2,7 @@
 from datetime import datetime
 from typing import Any, List
 from sqlalchemy import Enum as SAEnum, String
-from sqlalchemy import ForeignKey, func
+from sqlalchemy import ForeignKey, func, UniqueConstraint
 from sqlalchemy.orm import Mapped, mapped_column, DeclarativeBase, relationship
 from sqlalchemy.ext.asyncio import AsyncAttrs
 from sqlalchemy.types import JSON
@@ -26,6 +26,7 @@ class SubmissionDAO(Base):
     __tablename__ = "submission"
     id: Mapped[int] = mapped_column(index=True, primary_key=True, autoincrement=True)
     filing: Mapped[int] = mapped_column(ForeignKey("filing.id"))
+    counter: Mapped[int]
     submitter_id: Mapped[int] = mapped_column(ForeignKey("user_action.id"))
     submitter: Mapped[UserActionDAO] = relationship(lazy="selectin", foreign_keys=[submitter_id])
     accepter_id: Mapped[int] = mapped_column(ForeignKey("user_action.id"), nullable=True)
@@ -37,8 +38,10 @@ class SubmissionDAO(Base):
     filename: Mapped[str]
     total_records: Mapped[int] = mapped_column(nullable=True)
 
+    __table_args__ = (UniqueConstraint("filing", "counter", name="unique_filing_counter"),)
+
     def __str__(self):
-        return f"Submission ID: {self.id}, State: {self.state}, Ruleset: {self.validation_ruleset_version}, Filing Period: {self.filing}, Submission: {self.submission_time}"
+        return f"Submission ID: {self.id}, Counter: {self.counter}, State: {self.state}, Ruleset: {self.validation_ruleset_version}, Filing Period: {self.filing}, Submission: {self.submission_time}"
 
 
 class FilingPeriodDAO(Base):

src/sbl_filing_api/entities/models/dto.py

@@ -18,6 +18,7 @@ class SubmissionDTO(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
     id: int | None = None
+    counter: int
     state: SubmissionState | None = None
     validation_ruleset_version: str | None = None
     validation_results: Dict[str, Any] | None = None
@@ -27,6 +28,13 @@ class SubmissionDTO(BaseModel):
     submitter: UserActionDTO
     accepter: UserActionDTO | None = None
 
+    @model_validator(mode="after")
+    def validate_fi(self) -> "SubmissionDTO":
+        print(f"Self: {self}")
+        self.id = self.counter
+        print(f"Self: {self}")
+        return self
+
 
 class FilingTaskDTO(BaseModel):
     model_config = ConfigDict(from_attributes=True)

src/sbl_filing_api/entities/repos/submission_repo.py

@@ -56,6 +56,12 @@ async def get_submission(session: AsyncSession, submission_id: int) -> Submissio
     return result[0] if result else None
 
 
+async def get_submission_by_counter(session: AsyncSession, lei: str, filing_period: str, counter: int) -> SubmissionDAO:
+    filing = await get_filing(session, lei=lei, filing_period=filing_period)
+    result = await query_helper(session, SubmissionDAO, filing=filing.id, counter=counter)
+    return result[0] if result else None
+
+
 async def get_filing(session: AsyncSession, lei: str, filing_period: str) -> FilingDAO:
     result = await query_helper(session, FilingDAO, lei=lei, filing_period=filing_period)
     if result:
@@ -98,8 +104,15 @@ async def get_user_actions(session: AsyncSession) -> List[UserActionDAO]:
 
 
 async def add_submission(session: AsyncSession, filing_id: int, filename: str, submitter_id: int) -> SubmissionDAO:
+    stmt = select(SubmissionDAO).filter_by(filing=filing_id).order_by(desc(SubmissionDAO.counter)).limit(1)
+    last_sub = await session.scalar(stmt)
+    current_count = last_sub.counter if last_sub else 0
     new_sub = SubmissionDAO(
-        filing=filing_id, state=SubmissionState.SUBMISSION_STARTED, filename=filename, submitter_id=submitter_id
+        filing=filing_id,
+        state=SubmissionState.SUBMISSION_STARTED,
+        filename=filename,
+        submitter_id=submitter_id,
+        counter=(current_count + 1),
     )
     # this returns the attached object, most importantly with the new submission id
     new_sub = await session.merge(new_sub)

src/sbl_filing_api/routers/filing.py

@@ -1,5 +1,7 @@
 import asyncio
 import logging
+import csv
+import io
 
 from concurrent.futures import ProcessPoolExecutor
 from fastapi import Depends, Request, UploadFile, status
@@ -163,7 +165,9 @@ async def sign_filing(request: Request, lei: str, period_code: str):
             action_type=UserActionType.SIGN,
         ),
     )
-    filing.confirmation_id = lei + "-" + period_code + "-" + str(latest_sub.id) + "-" + str(sig.timestamp.timestamp())
+    filing.confirmation_id = (
+        lei + "-" + period_code + "-" + str(latest_sub.counter) + "-" + str(int(sig.timestamp.timestamp()))
+    )
     filing.signatures.append(sig)
     return await repo.upsert_filing(request.state.db_session, filing)
 
@@ -195,10 +199,13 @@ async def upload_file(request: Request, lei: str, period_code: str, file: Upload
         submission = await repo.add_submission(request.state.db_session, filing.id, file.filename, submitter.id)
         try:
             submission_processor.upload_to_storage(
-                period_code, lei, submission.id, content, file.filename.split(".")[-1]
+                period_code, lei, submission.counter, content, file.filename.split(".")[-1]
             )
 
             submission.state = SubmissionState.SUBMISSION_UPLOADED
+            with io.BytesIO(content) as byte_stream:
+                reader = csv.reader(io.TextIOWrapper(byte_stream))
+                submission.total_records = sum(1 for row in reader) - 1
             submission = await repo.update_submission(request.state.db_session, submission)
         except Exception as e:
             submission.state = SubmissionState.UPLOAD_FAILED
@@ -260,24 +267,24 @@ async def get_submission_latest(request: Request, lei: str, period_code: str):
     return Response(status_code=status.HTTP_204_NO_CONTENT)
 
 
-@router.get("/institutions/{lei}/filings/{period_code}/submissions/{id}", response_model=SubmissionDTO | None)
+@router.get("/institutions/{lei}/filings/{period_code}/submissions/{counter}", response_model=SubmissionDTO | None)
 @requires("authenticated")
-async def get_submission(request: Request, response: Response, id: int):
-    result = await repo.get_submission(request.state.db_session, id)
+async def get_submission(request: Request, response: Response, counter: int, lei: str, period_code: str):
+    result = await repo.get_submission_by_counter(request.state.db_session, lei, period_code, counter)
     if result:
         return result
     response.status_code = status.HTTP_404_NOT_FOUND
 
 
-@router.put("/institutions/{lei}/filings/{period_code}/submissions/{id}/accept", response_model=SubmissionDTO)
+@router.put("/institutions/{lei}/filings/{period_code}/submissions/{counter}/accept", response_model=SubmissionDTO)
 @requires("authenticated")
-async def accept_submission(request: Request, id: int, lei: str, period_code: str):
-    submission = await repo.get_submission(request.state.db_session, id)
+async def accept_submission(request: Request, counter: int, lei: str, period_code: str):
+    submission = await repo.get_submission_by_counter(request.state.db_session, lei, period_code, counter)
     if not submission:
         raise RegTechHttpException(
             status_code=status.HTTP_404_NOT_FOUND,
             name="Submission Not Found",
-            detail=f"Submission ID {id} does not exist, cannot accept a non-existing submission.",
+            detail=f"Submission {counter} for LEI {lei} in filing period {period_code} does not exist, cannot accept a non-existing submission.",
         )
     if (
         submission.state != SubmissionState.VALIDATION_SUCCESSFUL
@@ -286,7 +293,7 @@ async def accept_submission(request: Request, id: int, lei: str, period_code: st
         raise RegTechHttpException(
             status_code=status.HTTP_403_FORBIDDEN,
             name="Submission Action Forbidden",
-            detail=f"Submission {id} for LEI {lei} in filing period {period_code} is not in an acceptable state.  Submissions must be validated successfully or with only warnings to be accepted.",
+            detail=f"Submission {counter} for LEI {lei} in filing period {period_code} is not in an acceptable state.  Submissions must be validated successfully or with only warnings to be accepted.",
         )
 
     accepter = await repo.add_user_action(
@@ -368,13 +375,13 @@ async def get_latest_submission_report(request: Request, lei: str, period_code:
         SubmissionState.SUBMISSION_ACCEPTED,
     ]:
         file_data = submission_processor.get_from_storage(
-            period_code, lei, str(latest_sub.id) + submission_processor.REPORT_QUALIFIER
+            period_code, lei, str(latest_sub.counter) + submission_processor.REPORT_QUALIFIER
         )
         return StreamingResponse(
             content=file_data,
             media_type="text/csv",
             headers={
-                "Content-Disposition": f'attachment; filename="{latest_sub.id}_validation_report.csv"',
+                "Content-Disposition": f'attachment; filename="{latest_sub.counter}_validation_report.csv"',
                 "Cache-Control": "no-store",
             },
         )
@@ -387,34 +394,34 @@ async def get_latest_submission_report(request: Request, lei: str, period_code:
 
 
 @router.get(
-    "/institutions/{lei}/filings/{period_code}/submissions/{id}/report",
+    "/institutions/{lei}/filings/{period_code}/submissions/{counter}/report",
     responses={200: {"content": {"text/plain; charset=utf-8": {}}}},
 )
 @requires("authenticated")
-async def get_submission_report(request: Request, response: Response, lei: str, period_code: str, id: int):
-    sub = await repo.get_submission(request.state.db_session, id)
+async def get_submission_report(request: Request, response: Response, lei: str, period_code: str, counter: int):
+    sub = await repo.get_submission_by_counter(request.state.db_session, lei, period_code, counter)
     if sub and sub.state in [
         SubmissionState.VALIDATION_SUCCESSFUL,
         SubmissionState.VALIDATION_WITH_ERRORS,
         SubmissionState.VALIDATION_WITH_WARNINGS,
         SubmissionState.SUBMISSION_ACCEPTED,
     ]:
         file_data = submission_processor.get_from_storage(
-            period_code, lei, str(sub.id) + submission_processor.REPORT_QUALIFIER
+            period_code, lei, str(sub.counter) + submission_processor.REPORT_QUALIFIER
         )
         return StreamingResponse(
             content=file_data,
             media_type="text/csv",
             headers={
-                "Content-Disposition": f'attachment; filename="{sub.id}_validation_report.csv"',
+                "Content-Disposition": f'attachment; filename="{sub.counter}_validation_report.csv"',
                 "Cache-Control": "no-store",
             },
         )
     else:
         raise RegTechHttpException(
             status_code=status.HTTP_404_NOT_FOUND,
             name="Report Not Found",
-            detail=f"Report for ({id}) does not exist.",
+            detail=f"Report for ({lei}-{period_code}-{counter}) does not exist.",
         )