Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bug/catch-timeout-in-event-gather #157

Merged
merged 2 commits into from
Jan 27, 2022
Merged

bug/catch-timeout-in-event-gather #157

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
128ea0d
Catch ConnectionError in event gather pipeline
isaacna Jan 27, 2022
ee791de
Add in another catch and change to log.error
isaacna Jan 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 18 additions & 12 deletions cdp_backend/pipeline/event_gather_pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from gcsfs import GCSFileSystem
from prefect import Flow, task
from prefect.tasks.control_flow import case, merge
from requests import ConnectionError

from ..database import constants as db_constants
from ..database import functions as db_functions
Expand Down Expand Up @@ -1001,9 +1002,9 @@ def _process_person_ingestion(
db_model=person_picture_db_model,
credentials_file=credentials_file,
)
except (FileNotFoundError, ClientResponseError) as e:
except (FileNotFoundError, ClientResponseError, ConnectionError) as e:
person_picture_db_model = None
log.warning(
log.error(
f"Person ('{person.name}'), picture URI could not be archived."
f"({person.picture_uri}). Error: {e}"
)
Expand All @@ -1021,8 +1022,13 @@ def _process_person_ingestion(
db_model=person_db_model,
credentials_file=credentials_file,
)
except (FieldValidationFailed, RequiredField, InvalidFieldType):
log.warning(
except (
FieldValidationFailed,
RequiredField,
InvalidFieldType,
ConnectionError,
):
log.error(
f"Person ({person_db_model.to_dict()}), "
f"was missing required information. Generating minimum person details."
)
Expand Down Expand Up @@ -1069,9 +1075,9 @@ def _process_person_ingestion(
else:
person_seat_image_db_model = None

except (FileNotFoundError, ClientResponseError) as e:
except (FileNotFoundError, ClientResponseError, ConnectionError) as e:
person_seat_image_db_model = None
log.warning(
log.error(
f"Person ('{person.name}'), seat image URI could not be archived."
f"({person.seat.image_uri}). Error: {e}"
)
Expand Down Expand Up @@ -1240,8 +1246,8 @@ def store_event_processing_results(
db_model=event_db_model,
credentials_file=credentials_file,
)
except FieldValidationFailed:
log.warning(
except (FieldValidationFailed, ConnectionError):
log.error(
f"Agenda and/or minutes docs could not be found. "
f"Adding event without agenda and minutes URIs. "
f"({event.agenda_uri} AND/OR {event.minutes_uri} do not exist)"
Expand Down Expand Up @@ -1441,10 +1447,10 @@ def store_event_processing_results(
db_model=matter_file_db_model,
credentials_file=credentials_file,
)
except FieldValidationFailed:
except (FieldValidationFailed, ConnectionError) as e:
log.error(
f"MatterFile ('{supporting_file.uri}') "
f"could not be archived. Skipping."
f"could not be archived. Skipping. Error: {e}"
)

# Archive as event minutes item file
Expand All @@ -1460,10 +1466,10 @@ def store_event_processing_results(
db_model=event_minutes_item_file_db_model,
credentials_file=credentials_file,
)
except FieldValidationFailed:
except (FieldValidationFailed, ConnectionError) as e:
log.error(
f"EventMinutesItemFile ('{supporting_file.uri}') "
f"could not be archived."
f"could not be archived. Error: {e}"
)

# Add vote information
Expand Down