Catch ConnectionError in event gather pipeline

CouncilDataProject · Jan 27, 2022 · 128ea0d · 128ea0d
1 parent 4f8b6c7
commit 128ea0d
Showing 1 changed file with 14 additions and 8 deletions.
diff --git a/cdp_backend/pipeline/event_gather_pipeline.py b/cdp_backend/pipeline/event_gather_pipeline.py
@@ -14,6 +14,7 @@
 from gcsfs import GCSFileSystem
 from prefect import Flow, task
 from prefect.tasks.control_flow import case, merge
+from requests import ConnectionError
 
 from ..database import constants as db_constants
 from ..database import functions as db_functions
@@ -1001,7 +1002,7 @@ def _process_person_ingestion(
                     db_model=person_picture_db_model,
                     credentials_file=credentials_file,
                 )
-            except (FileNotFoundError, ClientResponseError) as e:
+            except (FileNotFoundError, ClientResponseError, ConnectionError) as e:
                 person_picture_db_model = None
                 log.warning(
                     f"Person ('{person.name}'), picture URI could not be archived."
@@ -1021,7 +1022,12 @@ def _process_person_ingestion(
                 db_model=person_db_model,
                 credentials_file=credentials_file,
             )
-        except (FieldValidationFailed, RequiredField, InvalidFieldType):
+        except (
+            FieldValidationFailed,
+            RequiredField,
+            InvalidFieldType,
+            ConnectionError,
+        ):
             log.warning(
                 f"Person ({person_db_model.to_dict()}), "
                 f"was missing required information. Generating minimum person details."
@@ -1069,7 +1075,7 @@ def _process_person_ingestion(
                 else:
                     person_seat_image_db_model = None
 
-            except (FileNotFoundError, ClientResponseError) as e:
+            except (FileNotFoundError, ClientResponseError, ConnectionError) as e:
                 person_seat_image_db_model = None
                 log.warning(
                     f"Person ('{person.name}'), seat image URI could not be archived."
@@ -1441,10 +1447,10 @@ def store_event_processing_results(
                                 db_model=matter_file_db_model,
                                 credentials_file=credentials_file,
                             )
-                        except FieldValidationFailed:
-                            log.error(
+                        except (FieldValidationFailed, ConnectionError) as e:
+                            log.warning(
                                 f"MatterFile ('{supporting_file.uri}') "
-                                f"could not be archived. Skipping."
+                                f"could not be archived. Skipping. Error: {e}"
                             )
 
                     # Archive as event minutes item file
@@ -1460,10 +1466,10 @@ def store_event_processing_results(
                             db_model=event_minutes_item_file_db_model,
                             credentials_file=credentials_file,
                         )
-                    except FieldValidationFailed:
+                    except (FieldValidationFailed, ConnectionError) as e:
                         log.error(
                             f"EventMinutesItemFile ('{supporting_file.uri}') "
-                            f"could not be archived."
+                            f"could not be archived. Error: {e}"
                         )
 
             # Add vote information