From 128ea0db31c44aa679dbadbeedd30b14a773faa9 Mon Sep 17 00:00:00 2001 From: isaac219 Date: Wed, 26 Jan 2022 19:21:24 -0800 Subject: [PATCH 1/2] Catch ConnectionError in event gather pipeline --- cdp_backend/pipeline/event_gather_pipeline.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/cdp_backend/pipeline/event_gather_pipeline.py b/cdp_backend/pipeline/event_gather_pipeline.py index 300320e5..826e2886 100644 --- a/cdp_backend/pipeline/event_gather_pipeline.py +++ b/cdp_backend/pipeline/event_gather_pipeline.py @@ -14,6 +14,7 @@ from gcsfs import GCSFileSystem from prefect import Flow, task from prefect.tasks.control_flow import case, merge +from requests import ConnectionError from ..database import constants as db_constants from ..database import functions as db_functions @@ -1001,7 +1002,7 @@ def _process_person_ingestion( db_model=person_picture_db_model, credentials_file=credentials_file, ) - except (FileNotFoundError, ClientResponseError) as e: + except (FileNotFoundError, ClientResponseError, ConnectionError) as e: person_picture_db_model = None log.warning( f"Person ('{person.name}'), picture URI could not be archived." @@ -1021,7 +1022,12 @@ def _process_person_ingestion( db_model=person_db_model, credentials_file=credentials_file, ) - except (FieldValidationFailed, RequiredField, InvalidFieldType): + except ( + FieldValidationFailed, + RequiredField, + InvalidFieldType, + ConnectionError, + ): log.warning( f"Person ({person_db_model.to_dict()}), " f"was missing required information. Generating minimum person details." @@ -1069,7 +1075,7 @@ def _process_person_ingestion( else: person_seat_image_db_model = None - except (FileNotFoundError, ClientResponseError) as e: + except (FileNotFoundError, ClientResponseError, ConnectionError) as e: person_seat_image_db_model = None log.warning( f"Person ('{person.name}'), seat image URI could not be archived." @@ -1441,10 +1447,10 @@ def store_event_processing_results( db_model=matter_file_db_model, credentials_file=credentials_file, ) - except FieldValidationFailed: - log.error( + except (FieldValidationFailed, ConnectionError) as e: + log.warning( f"MatterFile ('{supporting_file.uri}') " - f"could not be archived. Skipping." + f"could not be archived. Skipping. Error: {e}" ) # Archive as event minutes item file @@ -1460,10 +1466,10 @@ def store_event_processing_results( db_model=event_minutes_item_file_db_model, credentials_file=credentials_file, ) - except FieldValidationFailed: + except (FieldValidationFailed, ConnectionError) as e: log.error( f"EventMinutesItemFile ('{supporting_file.uri}') " - f"could not be archived." + f"could not be archived. Error: {e}" ) # Add vote information From ee791ded4fc21c9bb66bfa1820230a6f87004ebf Mon Sep 17 00:00:00 2001 From: isaac219 Date: Wed, 26 Jan 2022 19:40:01 -0800 Subject: [PATCH 2/2] Add in another catch and change to log.error --- cdp_backend/pipeline/event_gather_pipeline.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cdp_backend/pipeline/event_gather_pipeline.py b/cdp_backend/pipeline/event_gather_pipeline.py index 826e2886..ac6ea69d 100644 --- a/cdp_backend/pipeline/event_gather_pipeline.py +++ b/cdp_backend/pipeline/event_gather_pipeline.py @@ -1004,7 +1004,7 @@ def _process_person_ingestion( ) except (FileNotFoundError, ClientResponseError, ConnectionError) as e: person_picture_db_model = None - log.warning( + log.error( f"Person ('{person.name}'), picture URI could not be archived." f"({person.picture_uri}). Error: {e}" ) @@ -1028,7 +1028,7 @@ def _process_person_ingestion( InvalidFieldType, ConnectionError, ): - log.warning( + log.error( f"Person ({person_db_model.to_dict()}), " f"was missing required information. Generating minimum person details." ) @@ -1077,7 +1077,7 @@ def _process_person_ingestion( except (FileNotFoundError, ClientResponseError, ConnectionError) as e: person_seat_image_db_model = None - log.warning( + log.error( f"Person ('{person.name}'), seat image URI could not be archived." f"({person.seat.image_uri}). Error: {e}" ) @@ -1246,8 +1246,8 @@ def store_event_processing_results( db_model=event_db_model, credentials_file=credentials_file, ) - except FieldValidationFailed: - log.warning( + except (FieldValidationFailed, ConnectionError): + log.error( f"Agenda and/or minutes docs could not be found. " f"Adding event without agenda and minutes URIs. " f"({event.agenda_uri} AND/OR {event.minutes_uri} do not exist)" @@ -1448,7 +1448,7 @@ def store_event_processing_results( credentials_file=credentials_file, ) except (FieldValidationFailed, ConnectionError) as e: - log.warning( + log.error( f"MatterFile ('{supporting_file.uri}') " f"could not be archived. Skipping. Error: {e}" )