Feature/rename update type field names (#41)

rename update type field names lots of test data files renamed without change which is good as the updates made should be identical. --------- Co-authored-by: Mark <[email protected]>
climatepolicyradar · Apr 13, 2023 · 0a68e1a · 0a68e1a
1 parent 9a16c0a
commit 0a68e1a
Show file tree

Hide file tree

Showing 17 changed files with 56 additions and 56 deletions.
diff --git a/integration_tests/data/pipeline_in/input/new_and_updated_documents.json b/integration_tests/data/pipeline_in/input/new_and_updated_documents.json
@@ -939,41 +939,41 @@
     "updated_documents": {
        "TESTCCLW.executive.1.1": [
         {
-          "db_value": "name",
-          "csv_value": "new name",
+          "s3_value": "name",
+          "db_value": "new name",
           "type": "name"
         }
       ],
         "TESTCCLW.executive.2.2": [
         {
-          "db_value": "description",
-          "csv_value": "new description",
+          "s3_value": "description",
+          "db_value": "new description",
           "type": "description"
         }
       ],
       "TESTCCLW.executive.3.3": [
         {
-          "db_value": "http://existing.com",
-          "csv_value": "http://new.com",
+          "s3_value": "http://existing.com",
+          "db_value": "http://new.com",
           "type": "source_url"
         }
       ],
       "TESTCCLW.executive.4.4": [
         {
-          "db_value": "description",
-          "csv_value": "new description",
+          "s3_value": "description",
+          "db_value": "new description",
           "type": "description"
         },
         {
-          "db_value": "http://existing.com",
-          "csv_value": "http://new.com",
+          "s3_value": "http://existing.com",
+          "db_value": "http://new.com",
           "type": "source_url"
         }
       ],
       "TESTCCLW.executive.5.5": [
         {
-          "db_value": "description",
-          "csv_value": "new description",
+          "s3_value": "description",
+          "db_value": "new description",
           "type": "description"
         }
       ]

diff --git a/...W.executive.3.3/2023-04-12-13-01-01..json → ...W.executive.3.3/2023-04-13-13-25-36..json b/...W.executive.3.3/2023-04-12-13-01-01..json → ...W.executive.3.3/2023-04-13-13-25-36..json
diff --git a/...W.executive.4.4/2023-04-12-13-01-06..json → ...W.executive.4.4/2023-04-13-13-25-40..json b/...W.executive.4.4/2023-04-12-13-01-06..json → ...W.executive.4.4/2023-04-13-13-25-40..json
diff --git a/...CLW.executive.1.1/2023-04-12-13-01-05.npy → ...CLW.executive.1.1/2023-04-13-13-25-39.npy b/...CLW.executive.1.1/2023-04-12-13-01-05.npy → ...CLW.executive.1.1/2023-04-13-13-25-39.npy
diff --git a/...CLW.executive.2.2/2023-04-12-13-01-05.npy → ...CLW.executive.2.2/2023-04-13-13-25-39.npy b/...CLW.executive.2.2/2023-04-12-13-01-05.npy → ...CLW.executive.2.2/2023-04-13-13-25-39.npy
diff --git a/...W.executive.3.3/2023-04-12-13-01-01..json → ...W.executive.3.3/2023-04-13-13-25-36..json b/...W.executive.3.3/2023-04-12-13-01-01..json → ...W.executive.3.3/2023-04-13-13-25-36..json
diff --git a/...LW.executive.3.3/2023-04-12-13-01-01..npy → ...LW.executive.3.3/2023-04-13-13-25-36..npy b/...LW.executive.3.3/2023-04-12-13-01-01..npy → ...LW.executive.3.3/2023-04-13-13-25-36..npy
diff --git a/...W.executive.4.4/2023-04-12-13-01-06..json → ...W.executive.4.4/2023-04-13-13-25-40..json b/...W.executive.4.4/2023-04-12-13-01-06..json → ...W.executive.4.4/2023-04-13-13-25-40..json
diff --git a/...CLW.executive.4.4/2023-04-12-13-01-05.npy → ...CLW.executive.4.4/2023-04-13-13-25-40.npy b/...CLW.executive.4.4/2023-04-12-13-01-05.npy → ...CLW.executive.4.4/2023-04-13-13-25-40.npy
diff --git a/...W.executive.3.3/2023-04-12-13-01-01..json → ...W.executive.3.3/2023-04-13-13-25-36..json b/...W.executive.3.3/2023-04-12-13-01-01..json → ...W.executive.3.3/2023-04-13-13-25-36..json
diff --git a/...W.executive.4.4/2023-04-12-13-01-06..json → ...W.executive.4.4/2023-04-13-13-25-40..json b/...W.executive.4.4/2023-04-12-13-01-06..json → ...W.executive.4.4/2023-04-13-13-25-40..json
diff --git a/integration_tests/data/pipeline_out/input/new_and_updated_documents.json b/integration_tests/data/pipeline_out/input/new_and_updated_documents.json
@@ -939,41 +939,41 @@
     "updated_documents": {
        "TESTCCLW.executive.1.1": [
         {
-          "db_value": "name",
-          "csv_value": "new name",
+          "s3_value": "name",
+          "db_value": "new name",
           "type": "name"
         }
       ],
         "TESTCCLW.executive.2.2": [
         {
-          "db_value": "description",
-          "csv_value": "new description",
+          "s3_value": "description",
+          "db_value": "new description",
           "type": "description"
         }
       ],
       "TESTCCLW.executive.3.3": [
         {
-          "db_value": "http://existing.com",
-          "csv_value": "http://new.com",
+          "s3_value": "http://existing.com",
+          "db_value": "http://new.com",
           "type": "source_url"
         }
       ],
       "TESTCCLW.executive.4.4": [
         {
-          "db_value": "description",
-          "csv_value": "new description",
+          "s3_value": "description",
+          "db_value": "new description",
           "type": "description"
         },
         {
-          "db_value": "http://existing.com",
-          "csv_value": "http://new.com",
+          "s3_value": "http://existing.com",
+          "db_value": "http://new.com",
           "type": "source_url"
         }
       ],
       "TESTCCLW.executive.5.5": [
         {
-          "db_value": "description",
-          "csv_value": "new description",
+          "s3_value": "description",
+          "db_value": "new description",
           "type": "description"
         }
       ]

diff --git a/integration_tests/data/pipeline_out/input/new_and_updated_documents.json_errors b/integration_tests/data/pipeline_out/input/new_and_updated_documents.json_errors
diff --git a/src/navigator_data_ingest/base/types.py b/src/navigator_data_ingest/base/types.py
@@ -190,7 +190,7 @@ class Update(BaseModel):
     """Class describing the results of comparing csv data against the db data to identify updates."""
 
     db_value: Union[str, datetime]
-    csv_value: Union[str, datetime]
+    s3_value: Union[str, datetime]
     type: UpdateTypes
 
 

diff --git a/src/navigator_data_ingest/base/updated_document_actions.py b/src/navigator_data_ingest/base/updated_document_actions.py
@@ -161,8 +161,8 @@ def update_dont_parse(
                 update_file_field(
                     document_path=document_file,
                     field=str(document_update.type.value),
-                    new_value=document_update.csv_value,
-                    existing_value=document_update.db_value,
+                    new_value=document_update.db_value,
+                    existing_value=document_update.s3_value,
                 )
             )
 

diff --git a/src/navigator_data_ingest/tests/conftest.py b/src/navigator_data_ingest/tests/conftest.py
@@ -224,26 +224,26 @@ def test_updates(s3_document_id):
         s3_document_id: [
             {
                 "type": "name",
-                "csv_value": "NEW NAME",
-                "db_value": "An example document name.",
+                "db_value": "NEW NAME",
+                "s3_value": "An example document name.",
             },
             {
                 "type": "description",
-                "csv_value": "NEW DESCRIPTION",
-                "db_value": "An example document description.",
+                "db_value": "NEW DESCRIPTION",
+                "s3_value": "An example document description.",
             },
             {
                 "type": "source_url",
-                "csv_value": "https://www.NEW_SOURCE_URL.pdf",
-                "db_value": "https://domain/path/to/document.pdf",
+                "db_value": "https://www.NEW_SOURCE_URL.pdf",
+                "s3_value": "https://domain/path/to/document.pdf",
             },
         ]
     }
 
     return [
         Update(
             type=UpdateTypes(update["type"]),
-            csv_value=update["csv_value"],
+            s3_value=update["s3_value"],
             db_value=update["db_value"],
         )
         for update in updates[s3_document_id]

diff --git a/src/navigator_data_ingest/tests/test_update_actions.py b/src/navigator_data_ingest/tests/test_update_actions.py
@@ -125,15 +125,15 @@ def test_update_dont_parse(
         parser_input_doc_data[
             PipelineFieldMapping[UpdateTypes(update_to_document_description.type)]
         ]
-        == update_to_document_description.csv_value
+        == update_to_document_description.db_value
     )
 
     embeddings_input_doc_data = json.loads(embeddings_input_doc.read_text())
     assert (
         embeddings_input_doc_data[
             PipelineFieldMapping[UpdateTypes(update_to_document_description.type)]
         ]
-        == update_to_document_description.csv_value
+        == update_to_document_description.db_value
     )
 
     embeddings_input_translated_doc_data = json.loads(
@@ -143,15 +143,15 @@ def test_update_dont_parse(
         embeddings_input_translated_doc_data[
             PipelineFieldMapping[UpdateTypes(update_to_document_description.type)]
         ]
-        == update_to_document_description.csv_value
+        == update_to_document_description.db_value
     )
 
     indexer_input_doc_json_data = json.loads(indexer_input_doc_json.read_text())
     assert (
         indexer_input_doc_json_data[
             PipelineFieldMapping[UpdateTypes(update_to_document_description.type)]
         ]
-        == update_to_document_description.csv_value
+        == update_to_document_description.db_value
     )