Skip to content

Commit

Permalink
Merge pull request #26 from singer-io/refactor-since_id-bookmarking
Browse files Browse the repository at this point in the history
Rely on the same mechanism to both retrieve/reset the since_id bookmark
  • Loading branch information
dmosorast authored Jan 3, 2019
2 parents 886a785 + f6ff2dd commit 1b5de67
Showing 1 changed file with 5 additions and 7 deletions.
12 changes: 5 additions & 7 deletions tap_shopify/streams/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,16 @@ def call_api(self, query_params):
def get_objects(self):
updated_at_min = self.get_bookmark()

# Bookmarking can also occur on the since_id
since_id = self.get_since_id() or 1

if since_id != 1:
LOGGER.info("Resuming sync from since_id %d", since_id)

stop_time = singer.utils.now().replace(microsecond=0)
date_window_size = int(Context.config.get("date_window_size", DATE_WINDOW_SIZE))

# Page through till the end of the resultset
while updated_at_min < stop_time:
# Bookmarking can also occur on the since_id
since_id = self.get_since_id() or 1

if since_id != 1:
LOGGER.info("Resuming sync from since_id %d", since_id)

# It's important that `updated_at_min` has microseconds
# truncated. Why has been lost to the mists of time but we
Expand Down Expand Up @@ -167,7 +166,6 @@ def get_objects(self):
# restart at 1.
Context.state.get(self.name, {}).pop('since_id', None)
self.update_bookmark(utils.strftime(updated_at_max))
since_id = 1
break

if objects[-1].id != max([o.id for o in objects]):
Expand Down

0 comments on commit 1b5de67

Please sign in to comment.