diff --git a/metrics/sentry/cron.py b/metrics/sentry/cron.py new file mode 100644 index 0000000..f08b92b --- /dev/null +++ b/metrics/sentry/cron.py @@ -0,0 +1,58 @@ +import os + +import sentry_sdk +from sentry_sdk.crons import capture_checkin +from sentry_sdk.crons.consts import MonitorStatus + + +class Cron: + def __init__(self): + sentry_sdk.init( + dsn=os.environ.get("SENTRY_DSN"), + ) + + _monitor_config = { + "schedule": {"type": "crontab", "value": "@daily"}, + "timezone": "Etc/UTC", + # If an expected check-in doesn't come in `checkin_margin` + # minutes, it'll be considered missed + "checkin_margin": 30, + # The check-in is allowed to run for `max_runtime` minutes + # before it's considered failed + "max_runtime": 10, + # It'll take `failure_issue_threshold` consecutive failed + # check-ins to create an issue + "failure_issue_threshold": 1, + # It'll take `recovery_threshold` OK check-ins to resolve + # an issue + "recovery_threshold": 1, + } + + def get_monitor(self, modname): + monitor_slug = f"metrics-{modname}" + return self.Monitor(monitor_slug, self._monitor_config) + + class Monitor: + def __init__(self, monitor_slug, monitor_config): + self.monitor_slug = monitor_slug + self.monitor_config = monitor_config + self.check_in_id = None + + def _checkin(self, status): + check_in_id = capture_checkin( + monitor_slug=self.monitor_slug, + monitor_config=self.monitor_config, + status=status, + check_in_id=self.check_in_id, + ) + if not self.check_in_id: + self.check_in_id = check_in_id + + def in_progress(self): + self._checkin(status=MonitorStatus.IN_PROGRESS) + + def ok(self): + self._checkin(status=MonitorStatus.OK) + + def error(self): + self._checkin(status=MonitorStatus.ERROR) diff --git a/metrics/sentry/sentry.py b/metrics/sentry/sentry.py deleted file mode 100644 index 6a713bc..0000000 --- a/metrics/sentry/sentry.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -import sentry_sdk - - -def init(): - sentry_sdk.init( - dsn=os.environ.get("SENTRY_DSN"), - ) - - -monitor_config = { - "schedule": {"type": "crontab", "value": "@daily"}, - "timezone": "Etc/UTC", - # If an expected check-in doesn't come in `checkin_margin` - # minutes, it'll be considered missed - "checkin_margin": 30, - # The check-in is allowed to run for `max_runtime` minutes - # before it's considered failed - "max_runtime": 10, - # It'll take `failure_issue_threshold` consecutive failed - # check-ins to create an issue - "failure_issue_threshold": 1, - # It'll take `recovery_threshold` OK check-ins to resolve - # an issue - "recovery_threshold": 1, -} diff --git a/metrics/tasks/__main__.py b/metrics/tasks/__main__.py index bb9f7d4..43256aa 100644 --- a/metrics/tasks/__main__.py +++ b/metrics/tasks/__main__.py @@ -1,39 +1,26 @@ import pkgutil import structlog -from sentry_sdk.crons import capture_checkin -from sentry_sdk.crons.consts import MonitorStatus import metrics.tasks -from metrics.sentry import sentry +from metrics.sentry.cron import Cron log = structlog.get_logger() -sentry.init() +sentry_cron = Cron() for _, modname, _ in pkgutil.iter_modules(metrics.tasks.__path__): if modname != "__main__": log.info(f"Found {modname}") - monitor_slug = f"metrics-{modname}" - status = MonitorStatus.IN_PROGRESS - check_in_id = capture_checkin( - monitor_slug=monitor_slug, - status=status, - monitor_config=sentry.monitor_config, - ) + monitor = sentry_cron.get_monitor(modname) + monitor.in_progress() try: pkgutil.resolve_name(f"metrics.tasks.{modname}").main() - status = MonitorStatus.OK + monitor.ok() except AttributeError as error: log.error(f"Skipping {modname} because {error}") - status = MonitorStatus.ERROR + monitor.error() except Exception as exc: log.error(f"Failed to run {modname} because because an error occurred.") log.exception(exc) - status = MonitorStatus.ERROR - finally: - capture_checkin( - monitor_slug=monitor_slug, - status=status, - monitor_config=sentry.monitor_config, - ) + monitor.error()