From 122081fa97612b924e0a9dc3058ad44539a5e219 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Tue, 5 Mar 2024 10:58:11 -0500 Subject: [PATCH] source-gladly: don't emit a checkpoint with no documents every time Only emit an updated checkpoint when there are no documents if the current cursor is sufficiently old. Emitting a checkpoint with no documents will trigger an immediate re-invocation of the incremental task, so it shouldn't be done every time. --- source-gladly/source_gladly/api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source-gladly/source_gladly/api.py b/source-gladly/source_gladly/api.py index f7e0f4a9ee..16190d088e 100644 --- a/source-gladly/source_gladly/api.py +++ b/source-gladly/source_gladly/api.py @@ -60,6 +60,15 @@ async def fetch_events( f"not updating cursor since last_ts ({last_ts}) <= log_cursor ({log_cursor})" ) return + elif last_ts - log_cursor < timedelta(hours=6): + # Only emit an updated checkpoint when there are no documents if the current cursor is + # sufficiently old. Emitting a checkpoint with no documents will trigger an immediate + # re-invocation of this task, so it shouldn't be done every time. + log.debug( + f"not updating cursor since updated cursor is less than 6 hours newer than prior cursor (updated cursor: {last_ts} vs prior cursor: {log_cursor})" + ) + return + else: # The Events API has millisecond precision for timestamps, so bump up the log cursor one # millisecond to not re-fetch the last event seen again on the next round. The assumption