From f9460f78f4ab3490ea8a756efa524cbb72e30d95 Mon Sep 17 00:00:00 2001 From: Jabb0 <33359018+Jabb0@users.noreply.github.com> Date: Sat, 28 Nov 2020 19:59:47 +0100 Subject: [PATCH] Change to download now works without a limit again. --- psaw/PushshiftAPI.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/psaw/PushshiftAPI.py b/psaw/PushshiftAPI.py index 03efd52..8699e6c 100644 --- a/psaw/PushshiftAPI.py +++ b/psaw/PushshiftAPI.py @@ -214,14 +214,16 @@ def _handle_paging(self, url): data = self._get(url, self.payload) yield data - received_size = int(data['metadata']['size']) - requested_size = self.payload['limit'] - # Apparently the API can decide to send less data than desired. We need to send another request in that case - if received_size < requested_size: - limit += requested_size - received_size - - if (limit is not None) and (limit == 0): - return + if limit is not None: + received_size = int(data['metadata']['size']) + requested_size = self.payload['limit'] + # The API can decide to send less data than desired. + # We need to send another request in that case requesting the missing amount + if received_size < requested_size: + limit += requested_size - received_size + + if limit == 0: + return def _search(self, kind,