Feature/pdct 1350 ingest events

app/service/ingest.py

@@ -5,7 +5,7 @@
 import of data and other services for validation etc.
 """
 
-from typing import Optional
+from typing import Any, Optional
 
 from fastapi import HTTPException, status
 from pydantic import ConfigDict, validate_call
@@ -30,15 +30,17 @@
 
 @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
 def save_collections(
-    collection_data: list[dict], corpus_import_id: str, db: Optional[Session] = None
+    collection_data: list[dict[str, Any]],
+    corpus_import_id: str,
+    db: Optional[Session] = None,
 ) -> list[str]:
     """
     Creates new collections with the values passed.
 
-    :param list[dict] collection_data: The data to use for creating collections.
+    :param list[dict[str, Any]] collection_data: The data to use for creating collections.
     :param str corpus_import_id: The import_id of the corpus the collections belong to.
-    :param Optional[Session] The database session to use for saving collections.
-    :return str: The new import_ids for the saved collections.
+    :param Optional[Session] The database session to use for saving collections or None.
+    :return list[str]: The new import_ids for the saved collections.
     """
     if db is None:
         db = db_session.get_db()
@@ -58,15 +60,17 @@ def save_collections(
 
 @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
 def save_families(
-    family_data: list[dict], corpus_import_id: str, db: Optional[Session] = None
+    family_data: list[dict[str, Any]],
+    corpus_import_id: str,
+    db: Optional[Session] = None,
 ) -> list[str]:
     """
     Creates new families with the values passed.
 
-    :param list[dict] families_data: The data to use for creating families.
+    :param list[dict[str, Any]] families_data: The data to use for creating families.
     :param str corpus_import_id: The import_id of the corpus the families belong to.
-    :param Optional[Session] The database session to use for saving families.
-    :return str: The new import_ids for the saved families.
+    :param Optional[Session] The database session to use for saving families or None.
+    :return list[str]: The new import_ids for the saved families.
     """
 
     if db is None:
@@ -93,17 +97,17 @@ def save_families(
 
 @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
 def save_documents(
-    document_data: list[dict],
+    document_data: list[dict[str, Any]],
     corpus_import_id: str,
     db: Optional[Session] = None,
 ) -> list[str]:
     """
     Creates new documents with the values passed.
 
-    :param list[dict] document_data: The data to use for creating documents.
+    :param list[dict[str, Any]] document_data: The data to use for creating documents.
     :param str corpus_import_id: The import_id of the corpus the documents belong to.
-    :param Optional[Session] The database session to use for saving documents.
-    :return str: The new import_ids for the saved documents.
+    :param Optional[Session] The database session to use for saving documents or None.
+    :return list[str]: The new import_ids for the saved documents.
     """
     if db is None:
         db = db_session.get_db()
@@ -122,17 +126,17 @@ def save_documents(
 
 @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
 def save_events(
-    event_data: list[dict],
+    event_data: list[dict[str, Any]],
     corpus_import_id: str,
     db: Optional[Session] = None,
 ) -> list[str]:
     """
     Creates new events with the values passed.
 
-    :param list[dict] event_data: The data to use for creating events.
+    :param list[dict[str, Any]] event_data: The data to use for creating events.
     :param str corpus_import_id: The import_id of the corpus the events belong to.
-    :param Optional[Session] The database session to use for saving events.
-    :return str: The new import_ids for the saved events.
+    :param Optional[Session] The database session to use for saving events or None.
+    :return list[str]: The new import_ids for the saved events.
     """
     if db is None:
         db = db_session.get_db()
@@ -141,15 +145,22 @@ def save_events(
 
     event_import_ids = []
 
-    for ev in event_data:
-        dto = IngestEventDTO(**ev).to_event_create_dto()
+    for event in event_data:
+        dto = IngestEventDTO(**event).to_event_create_dto()
         import_id = event_repository.create(db, dto)
         event_import_ids.append(import_id)
 
     return event_import_ids
 
 
-def validate_entity_relationships(data: dict) -> None:
+def validate_entity_relationships(data: dict[str, Any]) -> None:
+    """
+    Validates relationships between entities contained in data based on import_ids.
+    For documents, it validates that the family the document is linked to exists.
+
+    :param dict[str, Any] data: The data object containing entities to be validated.
+    :raises ValidationError: raised should there be any unmatched relationships.
+    """
     families = []
     if "families" in data:
         for fam in data["families"]:
@@ -167,15 +178,15 @@ def validate_entity_relationships(data: dict) -> None:
 
 
 @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-def import_data(data: dict, corpus_import_id: str) -> dict:
+def import_data(data: dict[str, Any], corpus_import_id: str) -> dict[str, str]:
     """
     Imports data for a given corpus_import_id.
 
-    :param dict data: The data to be imported.
+    :param dict[str, Any] data: The data to be imported.
     :param str corpus_import_id: The import_id of the corpus the data should be imported into.
     :raises RepositoryError: raised on a database error.
-    :raises ValidationError: raised should the import_id be invalid.
-    :return dict: Import ids of the saved entities.
+    :raises ValidationError: raised should the data be invalid.
+    :return dict[str, str]: Import ids of the saved entities.
     """
     db = db_session.get_db()
 

app/service/validation.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Any, Optional
 
 from db_client.functions.corpus_helpers import TaxonomyData, get_taxonomy_from_corpus
 from db_client.models.dfce.taxonomy_entry import EntitySpecificTaxonomyKeys
@@ -12,16 +12,33 @@
 from app.service.collection import validate_import_id
 
 
-def validate_collection(collection: dict) -> None:
+def validate_collection(collection: dict[str, Any]) -> None:
+    """
+    Validates a collection.
+
+    :param dict[str, Any] collection: The collection object to be validated.
+    :raises ValidationError: raised should the data be invalid.
+    """
     validate_import_id(collection["import_id"])
 
 
-def validate_collections(collections: list[dict]) -> None:
+def validate_collections(collections: list[dict[str, Any]]) -> None:
+    """
+    Validates a list of collections.
+
+    :param list[dict[str, Any]] collections: The list of collection objects to be validated.
+    """
     for coll in collections:
         validate_collection(coll)
 
 
-def validate_family(family: dict, corpus_import_id: str) -> None:
+def validate_family(family: dict[str, Any], corpus_import_id: str) -> None:
+    """
+    Validates a family.
+
+    :param dict[str, Any] family: The family object to be validated.
+    :raises ValidationError: raised should the data be invalid.
+    """
     db = db_session.get_db()
 
     validate_import_id(family["import_id"])
@@ -32,12 +49,23 @@ def validate_family(family: dict, corpus_import_id: str) -> None:
     metadata.validate_metadata(db, corpus_import_id, family["metadata"])
 
 
-def validate_families(families: list[dict], corpus_import_id: str) -> None:
+def validate_families(families: list[dict[str, Any]], corpus_import_id: str) -> None:
+    """
+    Validates a list of families.
+
+    :param list[dict[str, Any]] families: The list of family objects to be validated.
+    """
     for fam in families:
         validate_family(fam, corpus_import_id)
 
 
-def validate_document(document: dict, corpus_import_id: str) -> None:
+def validate_document(document: dict[str, Any], corpus_import_id: str) -> None:
+    """
+    Validates a document.
+
+    :param dict[str, Any] document: The document object to be validated.
+    :raises ValidationError: raised should the data be invalid.
+    """
     db = db_session.get_db()
 
     validate_import_id(document["import_id"])
@@ -52,12 +80,23 @@ def validate_document(document: dict, corpus_import_id: str) -> None:
     )
 
 
-def validate_documents(documents: list[dict], corpus_import_id: str) -> None:
+def validate_documents(documents: list[dict[str, Any]], corpus_import_id: str) -> None:
+    """
+    Validates a list of documents.
+
+    :param list[dict[str, Any]] documents: The list of document objects to be validated.
+    """
     for doc in documents:
         validate_document(doc, corpus_import_id)
 
 
-def validate_event(event: dict, taxonomy: Optional[TaxonomyData]) -> None:
+def validate_event(event: dict[str, Any], taxonomy: Optional[TaxonomyData]) -> None:
+    """
+    Validates an event.
+
+    :param dict[str, Any] event: The event object to be validated.
+    :raises ValidationError: raised should the data be invalid.
+    """
     validate_import_id(event["import_id"])
     validate_import_id(event["family_import_id"])
     allowed_event_types = taxonomy["event_type"]["allowed_values"] if taxonomy else None
@@ -72,7 +111,12 @@ def validate_event(event: dict, taxonomy: Optional[TaxonomyData]) -> None:
         raise ValidationError(f"Event type ['{event['event_type_value']}'] is invalid!")
 
 
-def validate_events(events: list[dict], corpus_import_id: str) -> None:
+def validate_events(events: list[dict[str, Any]], corpus_import_id: str) -> None:
+    """
+    Validates a list of events.
+
+    :param list[dict[str, Any]] events: The list of event objects to be validated.
+    """
     db = db_session.get_db()
 
     event_taxonomy = get_taxonomy_from_corpus(db, corpus_import_id)

tests/integration_tests/ingest/test_ingest.py

@@ -12,7 +12,9 @@ def test_ingest_when_ok(data_db: Session, client: TestClient, user_header_token)
 
     response = client.post(
         "/api/v1/ingest/UNFCCC.corpus.i00000001.n0000",
-        files={"new_data": open("tests/integration_tests/ingest/test.json", "rb")},
+        files={
+            "new_data": open("tests/integration_tests/ingest/test_bulk_data.json", "rb")
+        },
         headers=user_header_token,
     )
 
@@ -79,7 +81,9 @@ def test_ingest_rollback(
 
     response = client.post(
         "/api/v1/ingest/UNFCCC.corpus.i00000001.n0000",
-        files={"new_data": open("tests/integration_tests/ingest/test.json", "rb")},
+        files={
+            "new_data": open("tests/integration_tests/ingest/test_bulk_data.json", "rb")
+        },
         headers=user_header_token,
     )
 
@@ -98,7 +102,9 @@ def test_ingest_when_corpus_import_id_invalid(
     invalid_corpus = "test"
     response = client.post(
         f"/api/v1/ingest/{invalid_corpus}",
-        files={"new_data": open("tests/integration_tests/ingest/test.json", "rb")},
+        files={
+            "new_data": open("tests/integration_tests/ingest/test_bulk_data.json", "rb")
+        },
         headers=user_header_token,
     )
 
@@ -116,7 +122,8 @@ def test_ingest_events_when_event_type_invalid(
         "/api/v1/ingest/UNFCCC.corpus.i00000001.n0000",
         files={
             "new_data": open(
-                "tests/integration_tests/ingest/test_invalid_event_type.json", "rb"
+                "tests/integration_tests/ingest/test_bulk_data_with_invalid_event_type.json",
+                "rb",
             )
         },
         headers=user_header_token,

tests/unit_tests/routers/ingest/test_ingest.py

@@ -31,7 +31,11 @@ def test_ingest_data_when_ok(
 
     response = client.post(
         "/api/v1/ingest/test",
-        files={"new_data": open("tests/unit_tests/routers/ingest/test.json", "rb")},
+        files={
+            "new_data": open(
+                "tests/unit_tests/routers/ingest/test_bulk_data.json", "rb"
+            )
+        },
         headers=user_header_token,
     )
 
@@ -94,7 +98,11 @@ def test_ingest_data_when_db_error(
 
     response = client.post(
         "/api/v1/ingest/test",
-        files={"new_data": open("tests/unit_tests/routers/ingest/test.json", "rb")},
+        files={
+            "new_data": open(
+                "tests/unit_tests/routers/ingest/test_bulk_data.json", "rb"
+            )
+        },
         headers=user_header_token,
     )