feat: Auto-detect end of pagination

meltano · Nov 17, 2023 · 15494ab · 15494ab
1 parent 2a64cf6
commit 15494ab
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py
@@ -395,7 +395,14 @@ def request_records(self, context: dict | None) -> t.Iterable[dict]:
                 resp = decorated_request(prepared_request, context)
                 request_counter.increment()
                 self.update_sync_costs(prepared_request, resp, context)
-                yield from self.parse_response(resp)
+                records = iter(self.parse_response(resp))
+                try:
+                    first_record = next(records)
+                except StopIteration:
+                    self.logger.info("No records found in response")
+                    break
+                yield first_record
+                yield from records
 
                 paginator.advance(resp)