Add support for iterating over lists with a generator.

README.md

@@ -60,14 +60,17 @@ access key can be found
 Simply add `count` and `offset` arguments in your function. The count
 is how many records to return, the offset is how many records to skip.
 For endpoints that allow the pagination parameters, the all() method
-has an additional boolean `get_all` argument that will loop through all
-records until the API no longer returns any to get all records without
-manually performing an additional query. By default, count is 10 and
-offset is 0 for all endpoints that support it. The `get_all` parameter
-on the all() method on any endpoint defaults to false, which follows
-the values that are provided in the call, and using `get_all=True` will
-ignore the provided count and offset to ensure that all records are
-returned. When using get_all, the count will be 5000, to fetch large
+has two additional booleans: `get_all` and `iterate`. By default
+`count` defaults to 10 and `offset` to 0. Setting either `get_all` or
+`iterate` to true will ignore both `count` and `offset`. Setting
+`iterate` and `get_all` both to true will result in a `ValueError`.
+
+The `iterate` argument will cause the method to return a generator 
+allowing the caller to iterate over each page without having to handle
+the pagination. 
+
+The `get_all` will collect all values and return them as a single result.
+When using get_all, the count will be 5000, to fetch large
 numbers of records without flooding the system with requests. The large
 size of count should not impact calls which are expected to return a
 very small number of records, and should improve performance for calls

README.rst

@@ -67,22 +67,25 @@ access key can be found
 Pagination
 ~~~~~~~~~~
 
-Simply add ``count`` and ``offset`` arguments in your function. The
-count is how many records to return, the offset is how many records to
-skip. For endpoints that allow the pagination parameters, the all()
-method has an additional boolean ``get_all`` argument that will loop
-through all records until the API no longer returns any to get all
-records without manually performing an additional query. By default,
-count is 10 and offset is 0 for all endpoints that support it. The
-``get_all`` parameter on the all() method on any endpoint defaults to
-false, which follows the values that are provided in the call, and using
-``get_all=True`` will ignore the provided count and offset to ensure
-that all records are returned. When using get_all, the count will be
-5000, to fetch large numbers of records without flooding the system with
-requests. The large size of count should not impact calls which are
-expected to return a very small number of records, and should improve
-performance for calls where fetching 5000 records would only provide a
-fraction by preventing the delay of making a huge number of requests.
+Simply add ``count`` and ``offset`` arguments in your function. The count
+is how many records to return, the offset is how many records to skip.
+For endpoints that allow the pagination parameters, the all() method
+has two additional booleans: ``get_all`` and ``iterate``. By default
+``count`` defaults to 10 and ``offset`` to 0. Setting either ``get_all`` or
+``iterate`` to true will ignore both ``count`` and ``offset``. Setting
+``iterate`` and ``get_all`` both to true will result in a ``ValueError``.
+
+The ``iterate`` argument will cause the method to return a generator
+allowing the caller to iterate over each page without having to handle
+the pagination.
+
+The ``get_all`` will collect all values and return them as a single result.
+When using get_all, the count will be 5000, to fetch large
+numbers of records without flooding the system with requests. The large
+size of count should not impact calls which are expected to return a
+very small number of records, and should improve performance for calls
+where fetching 5000 records would only provide a fraction by preventing
+the delay of making a huge number of requests.
 
 ::
 

mailchimp3/baseapi.py

@@ -33,9 +33,27 @@ def _build_path(self, *args):
         """
         return '/'.join(chain((self.endpoint,), map(str, args)))
 
+    def _list_result(self, url, get_all, iterate, **queryparams):
+        """
+        Simplify list by returning the single page, all pages or iterates.
+
+        :param get_all: Should the query get all results
+        :type get_all: :py:class:`bool`
+        :param iterate: Should the query iterate over each page.
+        :type iterate: :py:class:`bool`
+        """
+        if get_all and iterate:
+            raise ValueError("Both get_all and iterate can't be True.")
+        if get_all:
+            return self._all(url=url, **queryparams)
+        elif iterate:
+            return self._iterate(url=url, **queryparams)
+        else:
+            return self._mc_client._get(url=url, **queryparams)
+
     def _iterate(self, url, **queryparams):
         """
-        Iterate over all pages for the given url. Feed in the result of self._build_path as the url.
+        Iterate over pages for the given url. Feed in the result of self._build_path as the url.
 
         :param url: The url of the endpoint
         :type url: :py:class:`str`
@@ -57,17 +75,36 @@ def _iterate(self, url, **queryparams):
         queryparams.pop("offset", None)
         queryparams.pop("count", None)
         # Fetch results from mailchimp, up to first 1000
-        result = self._mc_client._get(url=url, offset=0, count=1000, **queryparams)
+        page_size = 1000
+        result = self._mc_client._get(url=url, offset=0, count=page_size, **queryparams)
         total = result['total_items']
         # Fetch further results if necessary
-        if total > 1000:
-            for offset in range(1, int(total / 1000) + 1):
-                result = merge_results(result, self._mc_client._get(
+        if total > page_size:
+            yield result
+            for offset in range(1, int(total / page_size)):
+                yield self._mc_client._get(
                     url=url,
-                    offset=int(offset * 1000),
-                    count=1000,
+                    offset=int(offset * page_size),
+                    count=page_size,
                     **queryparams
-                ))
-            return result
+                )
         else:  # Further results not necessary
-            return result
+            yield result
+
+    def _all(self, url, **queryparams):
+        """
+        Iterate over all pages for the given url and return a single collection.
+        Feed in the result of self._build_path as the url.
+
+        :param url: The url of the endpoint
+        :type url: :py:class:`str`
+        :param queryparams: The query string parameters
+        queryparams['fields'] = []
+        queryparams['exclude_fields'] = []
+        queryparams['count'] = integer
+        queryparams['offset'] = integer
+        """
+        result = {}
+        for page in self._iterate(url, **queryparams):
+            result = merge_results(result, page)
+        return result

mailchimp3/entities/authorizedapps.py

@@ -44,23 +44,22 @@ def create(self, data):
         return self._mc_client._post(url=self._build_path(), data=data)
 
 
-    def all(self, get_all=False, **queryparams):
+    def all(self, get_all=False, iterate=False, **queryparams):
         """
         Get a list of an account’s registered, connected applications.
 
         :param get_all: Should the query get all results
         :type get_all: :py:class:`bool`
+        :param iterate: Should the query iterate over each page.
+        :type iterate: :py:class:`bool`
         :param queryparams: The query string parameters
         queryparams['fields'] = []
         queryparams['exclude_fields'] = []
         queryparams['count'] = integer
         queryparams['offset'] = integer
         """
         self.app_id = None
-        if get_all:
-            return self._iterate(url=self._build_path(), **queryparams)
-        else:
-            return self._mc_client._get(url=self._build_path(), **queryparams)
+        return self._list_result(self._build_path(), get_all, iterate, **queryparams)
 
 
     def get(self, app_id, **queryparams):

mailchimp3/entities/automationemails.py

@@ -31,24 +31,24 @@ def __init__(self, *args, **kwargs):
 
 
     # Paid feature
-    def all(self, workflow_id, get_all=False, **queryparams):
+    def all(self, workflow_id, get_all=False, iterate=False, **queryparams):
         """
         Get a summary of the emails in an Automation workflow.
 
         :param workflow_id: The unique id for the Automation workflow.
         :type workflow_id: :py:class:`str`
         :param get_all: Should the query get all results
         :type get_all: :py:class:`bool`
+        :param iterate: Should the query iterate over each page.
+        :type iterate: :py:class:`bool`
         :param queryparams: the query string parameters
         queryparams['fields'] = []
         queryparams['exclude_fields'] = []
         """
         self.workflow_id = workflow_id
         self.email_id = None
-        if get_all:
-            return self._iterate(url=self._build_path(workflow_id, 'emails'), **queryparams)
-        else:
-            return self._mc_client._get(url=self._build_path(workflow_id, 'emails'), **queryparams)
+        url = self._build_path(workflow_id, 'emails')
+        return self._list_result(url, get_all, iterate, **queryparams)
 
 
     # Paid feature

mailchimp3/entities/automations.py

@@ -34,21 +34,20 @@ def __init__(self, *args, **kwargs):
 
 
     # Paid feature
-    def all(self, get_all=False, **queryparams):
+    def all(self, get_all=False, iterate=False, **queryparams):
         """
         Get a summary of an account’s Automations.
 
         :param get_all: Should the query get all results
         :type get_all: :py:class:`bool`
+        :param iterate: Should the query iterate over each page.
+        :type iterate: :py:class:`bool`
         :param queryparams: the query string parameters
         queryparams['fields'] = []
         queryparams['exclude_fields'] = []
         """
         self.workflow_id = None
-        if get_all:
-            return self._iterate(url=self._build_path(), **queryparams)
-        else:
-            return self._mc_client._get(url=self._build_path(), **queryparams)
+        return self._list_result(self._build_path(), get_all, iterate, **queryparams)
 
 
     # Paid feature

mailchimp3/entities/batchoperations.py

@@ -53,12 +53,14 @@ def create(self, data):
         return self._mc_client._post(url=self._build_path(), data=data)
 
 
-    def all(self, get_all=False, **queryparams):
+    def all(self, get_all=False, iterate=False, **queryparams):
         """
         Get a summary of batch requests that have been made.
 
         :param get_all: Should the query get all results
         :type get_all: :py:class:`bool`
+        :param iterate: Should the query iterate over each page.
+        :type iterate: :py:class:`bool`
         :param queryparams: The query string parameters
         queryparams['fields'] = []
         queryparams['exclude_fields'] = []
@@ -67,10 +69,7 @@ def all(self, get_all=False, **queryparams):
         """
         self.batch_id = None
         self.operation_status = None
-        if get_all:
-            return self._iterate(url=self._build_path(), **queryparams)
-        else:
-            return self._mc_client._get(url=self._build_path(), **queryparams)
+        return self._list_result(self._build_path(), get_all, iterate, **queryparams)
 
 
     def get(self, batch_id, **queryparams):

mailchimp3/entities/batchwebhooks.py

@@ -43,23 +43,22 @@ def create(self, data):
         return response
 
 
-    def all(self, get_all=False, **queryparams):
+    def all(self, get_all=False, iterate=False, **queryparams):
         """
         Get all webhooks that have been configured for batches.
 
         :param get_all: Should the query get all results
         :type get_all: :py:class:`bool`
+        :param iterate: Should the query iterate over each page.
+        :type iterate: :py:class:`bool`
         :param queryparams: The query string parameters
         queryparams['fields'] = []
         queryparams['exclude_fields'] = []
         queryparams['count'] = integer
         queryparams['offset'] = integer
         """
         self.batch_webhook_id = None
-        if get_all:
-            return self._iterate(url=self._build_path(), **queryparams)
-        else:
-            return self._mc_client._get(url=self._build_path(), **queryparams)
+        return self._list_result(self._build_path(), get_all, iterate, **queryparams)
 
 
     def get(self, batch_webhook_id, **queryparams):

mailchimp3/entities/campaignfeedback.py

@@ -51,7 +51,7 @@ def create(self, campaign_id, data, **queryparams):
         return response
 
 
-    def all(self, campaign_id, get_all=False, **queryparams):
+    def all(self, campaign_id, get_all=False, iterate=False, **queryparams):
         """
         Get team feedback while you’re working together on a MailChimp
         campaign.
@@ -60,16 +60,17 @@ def all(self, campaign_id, get_all=False, **queryparams):
         :type campaign_id: :py:class:`str`
         :param get_all: Should the query get all results
         :type get_all: :py:class:`bool`
+        :param iterate: Should the query iterate over each page.
+        :type iterate: :py:class:`bool`
         :param queryparams: The query string parameters
         queryparams['fields'] = []
         queryparams['exclude_fields'] = []
         """
         self.campaign_id = campaign_id
         self.feedback_id = None
-        if get_all:
-            return self._iterate(url=self._build_path(campaign_id, 'feedback'), **queryparams)
-        else:
-            return self._mc_client._get(url=self._build_path(campaign_id, 'feedback'), **queryparams)
+
+        url = self._build_path(campaign_id, 'feedback')
+        return self._list_result(url, get_all, iterate, **queryparams)
 
 
     def get(self, campaign_id, feedback_id, **queryparams):

mailchimp3/entities/campaignfolders.py

@@ -43,23 +43,22 @@ def create(self, data):
         return response
 
 
-    def all(self, get_all=False, **queryparams):
+    def all(self, get_all=False, iterate=False, **queryparams):
         """
         Get all folders used to organize campaigns.
 
         :param get_all: Should the query get all results
         :type get_all: :py:class:`bool`
+        :param iterate: Should the query iterate over each page.
+        :type iterate: :py:class:`bool`
         :param queryparams: The query string parameters
         queryparams['fields'] = []
         queryparams['exclude_fields'] = []
         queryparams['count'] = integer
         queryparams['offset'] = integer
         """
         self.folder_id = None
-        if get_all:
-            return self._iterate(url=self._build_path(), **queryparams)
-        else:
-            return self._mc_client._get(url=self._build_path(), **queryparams)
+        return self._list_result(self._build_path(), get_all, iterate, **queryparams)
 
 
     def get(self, folder_id, **queryparams):

mailchimp3/entities/campaigns.py

@@ -106,7 +106,7 @@ def create(self, data):
         return response
 
 
-    def all(self, get_all=False, **queryparams):
+    def all(self, get_all=False, iterate=False, **queryparams):
         """
         Get all campaigns in an account.
 
@@ -117,6 +117,8 @@ def all(self, get_all=False, **queryparams):
 
         :param get_all: Should the query get all results
         :type get_all: :py:class:`bool`
+        :param iterate: Should the query iterate over each page.
+        :type iterate: :py:class:`bool`
         :param queryparams: The query string parameters
         queryparams['fields'] = []
         queryparams['exclude_fields'] = []
@@ -134,10 +136,7 @@ def all(self, get_all=False, **queryparams):
         queryparams['sort_dir'] = string
         """
         self.campaign_id = None
-        if get_all:
-            return self._iterate(url=self._build_path(), **queryparams)
-        else:
-            return self._mc_client._get(url=self._build_path(), **queryparams)
+        return self._list_result(self._build_path(), get_all, iterate, **queryparams)
 
 
     def get(self, campaign_id, **queryparams):

mailchimp3/entities/conversations.py

@@ -29,12 +29,14 @@ def __init__(self, *args, **kwargs):
 
 
     # Paid feature
-    def all(self, get_all=False, **queryparams):
+    def all(self, get_all=False, iterate=False, **queryparams):
         """
         Get a list of conversations for the account.
 
         :param get_all: Should the query get all results
         :type get_all: :py:class:`bool`
+        :param iterate: Should the query iterate over each page.
+        :type iterate: :py:class:`bool`
         :param queryparams: The query string parameters
         queryparams['fields'] = []
         queryparams['exclude_fields'] = []
@@ -45,10 +47,7 @@ def all(self, get_all=False, **queryparams):
         queryparams['campaign_id'] = string
         """
         self.conversation_id = None
-        if get_all:
-            return self._iterate(url=self._build_path(), **queryparams)
-        else:
-            return self._mc_client._get(url=self._build_path(), **queryparams)
+        return self._list_result(self._build_path(), get_all, iterate, **queryparams)
 
 
     # Paid feature