diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index deb83516b9d7..9358229e630a 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -684,7 +684,7 @@ def _item_to_row(iterator, resource): added to the iterator after being created, which should be done by the caller. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict @@ -700,7 +700,7 @@ def _item_to_row(iterator, resource): def _rows_page_start(iterator, page, response): """Grab total rows when :class:`~google.cloud.iterator.Page` starts. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type page: :class:`~google.cloud.iterator.Page` diff --git a/bigquery/google/cloud/bigquery/client.py b/bigquery/google/cloud/bigquery/client.py index f36d80978efd..d9ff17d71720 100644 --- a/bigquery/google/cloud/bigquery/client.py +++ b/bigquery/google/cloud/bigquery/client.py @@ -14,7 +14,7 @@ """Client for interacting with the Google BigQuery API.""" - +from google.api.core import page_iterator from google.cloud.client import ClientWithProject from google.cloud.bigquery._http import Connection from google.cloud.bigquery.dataset import Dataset @@ -23,7 +23,6 @@ from google.cloud.bigquery.job import LoadTableFromStorageJob from google.cloud.bigquery.job import QueryJob from google.cloud.bigquery.query import QueryResults -from google.cloud.iterator import HTTPIterator class Project(object): @@ -98,13 +97,17 @@ def list_projects(self, max_results=None, page_token=None): not passed, the API will return the first page of projects. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.bigquery.client.Project` accessible to the current client. """ - return HTTPIterator( - client=self, path='/projects', item_to_value=_item_to_project, - items_key='projects', page_token=page_token, + return page_iterator.HTTPIterator( + client=self, + api_request=self._connection.api_request, + path='/projects', + item_to_value=_item_to_project, + items_key='projects', + page_token=page_token, max_results=max_results) def list_datasets(self, include_all=False, max_results=None, @@ -126,7 +129,7 @@ def list_datasets(self, include_all=False, max_results=None, not passed, the API will return the first page of datasets. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.bigquery.dataset.Dataset`. accessible to the current client. """ @@ -134,10 +137,15 @@ def list_datasets(self, include_all=False, max_results=None, if include_all: extra_params['all'] = True path = '/projects/%s/datasets' % (self.project,) - return HTTPIterator( - client=self, path=path, item_to_value=_item_to_dataset, - items_key='datasets', page_token=page_token, - max_results=max_results, extra_params=extra_params) + return page_iterator.HTTPIterator( + client=self, + api_request=self._connection.api_request, + path=path, + item_to_value=_item_to_dataset, + items_key='datasets', + page_token=page_token, + max_results=max_results, + extra_params=extra_params) def dataset(self, dataset_name, project=None): """Construct a dataset bound to this client. @@ -207,7 +215,7 @@ def list_jobs(self, max_results=None, page_token=None, all_users=None, * ``"pending"`` * ``"running"`` - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterable of job instances. """ extra_params = {'projection': 'full'} @@ -219,10 +227,15 @@ def list_jobs(self, max_results=None, page_token=None, all_users=None, extra_params['stateFilter'] = state_filter path = '/projects/%s/jobs' % (self.project,) - return HTTPIterator( - client=self, path=path, item_to_value=_item_to_job, - items_key='jobs', page_token=page_token, - max_results=max_results, extra_params=extra_params) + return page_iterator.HTTPIterator( + client=self, + api_request=self._connection.api_request, + path=path, + item_to_value=_item_to_job, + items_key='jobs', + page_token=page_token, + max_results=max_results, + extra_params=extra_params) def load_table_from_storage(self, job_name, destination, *source_uris): """Construct a job for loading data into a table from CloudStorage. @@ -349,7 +362,7 @@ def run_sync_query(self, query, udf_resources=(), query_parameters=()): def _item_to_project(iterator, resource): """Convert a JSON project to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict @@ -365,7 +378,7 @@ def _item_to_project(iterator, resource): def _item_to_dataset(iterator, resource): """Convert a JSON dataset to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict @@ -380,7 +393,7 @@ def _item_to_dataset(iterator, resource): def _item_to_job(iterator, resource): """Convert a JSON job to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict diff --git a/bigquery/google/cloud/bigquery/dataset.py b/bigquery/google/cloud/bigquery/dataset.py index 1304d5028873..d25f6747285f 100644 --- a/bigquery/google/cloud/bigquery/dataset.py +++ b/bigquery/google/cloud/bigquery/dataset.py @@ -15,10 +15,10 @@ """Define API Datasets.""" import six +from google.api.core import page_iterator from google.cloud._helpers import _datetime_from_microseconds from google.cloud.exceptions import NotFound from google.cloud.bigquery.table import Table -from google.cloud.iterator import HTTPIterator class AccessGrant(object): @@ -561,14 +561,19 @@ def list_tables(self, max_results=None, page_token=None): datasets. If not passed, the API will return the first page of datasets. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.bigquery.table.Table` contained within the current dataset. """ path = '/projects/%s/datasets/%s/tables' % (self.project, self.name) - result = HTTPIterator(client=self._client, path=path, - item_to_value=_item_to_table, items_key='tables', - page_token=page_token, max_results=max_results) + result = page_iterator.HTTPIterator( + client=self._client, + api_request=self._client._connection.api_request, + path=path, + item_to_value=_item_to_table, + items_key='tables', + page_token=page_token, + max_results=max_results) result.dataset = self return result @@ -590,7 +595,7 @@ def table(self, name, schema=()): def _item_to_table(iterator, resource): """Convert a JSON table to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict diff --git a/bigquery/google/cloud/bigquery/query.py b/bigquery/google/cloud/bigquery/query.py index 502953b2c828..dfa0a422a68a 100644 --- a/bigquery/google/cloud/bigquery/query.py +++ b/bigquery/google/cloud/bigquery/query.py @@ -16,7 +16,7 @@ import six -from google.cloud.iterator import HTTPIterator +from google.api.core import page_iterator from google.cloud.bigquery._helpers import _TypedProperty from google.cloud.bigquery._helpers import _rows_from_json from google.cloud.bigquery.dataset import Dataset @@ -414,7 +414,7 @@ def fetch_data(self, max_results=None, page_token=None, start_index=None, :param client: the client to use. If not passed, falls back to the ``client`` stored on the current dataset. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of row data :class:`tuple`s. During each page, the iterator will have the ``total_rows`` attribute set, which counts the total number of rows **in the result @@ -435,13 +435,16 @@ def fetch_data(self, max_results=None, page_token=None, start_index=None, params['timeoutMs'] = timeout_ms path = '/projects/%s/queries/%s' % (self.project, self.name) - iterator = HTTPIterator(client=client, path=path, - item_to_value=_item_to_row, - items_key='rows', - page_token=page_token, - max_results=max_results, - page_start=_rows_page_start_query, - extra_params=params) + iterator = page_iterator.HTTPIterator( + client=client, + api_request=client._connection.api_request, + path=path, + item_to_value=_item_to_row, + items_key='rows', + page_token=page_token, + max_results=max_results, + page_start=_rows_page_start_query, + extra_params=params) iterator.query_result = self # Over-ride the key used to retrieve the next page token. iterator._NEXT_TOKEN = 'pageToken' @@ -457,7 +460,7 @@ def _rows_page_start_query(iterator, page, response): added to the iterator after being created, which should be done by the caller. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type page: :class:`~google.cloud.iterator.Page` diff --git a/bigquery/google/cloud/bigquery/table.py b/bigquery/google/cloud/bigquery/table.py index c6bf5db893ab..87cff2980c7e 100644 --- a/bigquery/google/cloud/bigquery/table.py +++ b/bigquery/google/cloud/bigquery/table.py @@ -23,10 +23,10 @@ from google.resumable_media.requests import MultipartUpload from google.resumable_media.requests import ResumableUpload +from google.api.core import page_iterator from google.cloud import exceptions from google.cloud._helpers import _datetime_from_microseconds from google.cloud._helpers import _millis_from_datetime -from google.cloud.iterator import HTTPIterator from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery._helpers import _item_to_row from google.cloud.bigquery._helpers import _rows_page_start @@ -712,7 +712,7 @@ def fetch_data(self, max_results=None, page_token=None, client=None): :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current dataset. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of row data :class:`tuple`s. During each page, the iterator will have the ``total_rows`` attribute set, which counts the total number of rows **in the table** @@ -724,10 +724,15 @@ def fetch_data(self, max_results=None, page_token=None, client=None): client = self._require_client(client) path = '%s/data' % (self.path,) - iterator = HTTPIterator(client=client, path=path, - item_to_value=_item_to_row, items_key='rows', - page_token=page_token, max_results=max_results, - page_start=_rows_page_start) + iterator = page_iterator.HTTPIterator( + client=client, + api_request=client._connection.api_request, + path=path, + item_to_value=_item_to_row, + items_key='rows', + page_token=page_token, + max_results=max_results, + page_start=_rows_page_start) iterator.schema = self._schema # Over-ride the key used to retrieve the next page token. iterator._NEXT_TOKEN = 'pageToken' diff --git a/core/google/cloud/iterator.py b/core/google/api/core/page_iterator.py similarity index 52% rename from core/google/cloud/iterator.py rename to core/google/api/core/page_iterator.py index 742443ddc5f9..147c9f47e35a 100644 --- a/core/google/cloud/iterator.py +++ b/core/google/api/core/page_iterator.py @@ -12,56 +12,49 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Iterators for paging through API responses. +"""Iterators for paging through paged API methods. These iterators simplify the process of paging through API responses -where the response is a list of results with a ``nextPageToken``. - -To make an iterator work, you'll need to provide a way to convert a JSON -item returned from the API into the object of your choice (via -``item_to_value``). You also may need to specify a custom ``items_key`` so -that a given response (containing a page of results) can be parsed into an -iterable page of the actual objects you want. You then can use this to get -**all** the results from a resource:: - - >>> def item_to_value(iterator, item): - ... my_item = MyItemClass(iterator.client, other_arg=True) - ... my_item._set_properties(item) - ... return my_item - ... - >>> iterator = Iterator(..., items_key='blocks', - ... item_to_value=item_to_value) - >>> list(iterator) # Convert to a list (consumes all values). +where the request takes a page token and the response is a list of results with +a token for the next page. See `list pagination`_ in the Google API Style Guide +for more details. + +.. _list pagination: + https://cloud.google.com/apis/design/design_patterns#list_pagination + +API clients that have methods that follow the list pagination pattern can +return an :class:`Iterator`. You can use this iterator to get **all** of +the results across all pages:: + + >>> results_iterator = client.list_resources() + >>> list(results_iterator) # Convert to a list (consumes all values). Or you can walk your way through items and call off the search early if -you find what you're looking for (resulting in possibly fewer -requests):: +you find what you're looking for (resulting in possibly fewer requests):: - >>> for my_item in Iterator(...): - ... print(my_item.name) - ... if not my_item.is_valid: + >>> for resource in results_iterator: + ... print(resource.name) + ... if not resource.is_valid: ... break At any point, you may check the number of items consumed by referencing the ``num_results`` property of the iterator:: - >>> my_iterator = Iterator(...) - >>> for my_item in my_iterator: - ... if my_iterator.num_results >= 10: + >>> for my_item in results_iterator: + ... if results_iterator.num_results >= 10: ... break When iterating, not every new item will send a request to the server. To iterate based on each page of items (where a page corresponds to a request):: - >>> iterator = Iterator(...) - >>> for page in iterator.pages: + >>> for page in results_iterator.pages: ... print('=' * 20) - ... print(' Page number: %d' % (iterator.page_number,)) - ... print(' Items in page: %d' % (page.num_items,)) - ... print(' First item: %r' % (next(page),)) - ... print('Items remaining: %d' % (page.remaining,)) - ... print('Next page token: %s' % (iterator.next_page_token,)) + ... print(' Page number: {:d}'.format(iterator.page_number)) + ... print(' Items in page: {:d}'.format(page.num_items)) + ... print(' First item: {!r}'.format(next(page))) + ... print('Items remaining: {:d}'.format(page.remaining)) + ... print('Next page token: {}'.format(iterator.next_page_token)) ==================== Page number: 1 Items in page: 1 @@ -75,7 +68,8 @@ Items remaining: 18 Next page token: None -To consume an entire page:: +Then, for each page you can get all the resources on that page by iterating +through it or using :func:`list`:: >>> list(page) [ @@ -85,47 +79,21 @@ ] """ +import abc import six -DEFAULT_ITEMS_KEY = 'items' -"""The dictionary key used to retrieve items from each response.""" - - -# pylint: disable=unused-argument -def _do_nothing_page_start(iterator, page, response): - """Helper to provide custom behavior after a :class:`Page` is started. - - This is a do-nothing stand-in as the default value. - - :type iterator: :class:`Iterator` - :param iterator: An iterator that holds some request info. - - :type page: :class:`Page` - :param page: The page that was just created. - - :type response: dict - :param response: The JSON API response for a page. - """ -# pylint: enable=unused-argument - - class Page(object): """Single page of results in an iterator. - :type parent: :class:`Iterator` - :param parent: The iterator that owns the current page. - - :type items: iterable - :param items: An iterable (that also defines __len__) of items - from a raw API response. - - :type item_to_value: callable - :param item_to_value: Callable to convert an item from the type in the - raw API response into the native object. - Assumed signature takes an :class:`Iterator` and a - raw API response with a single item. + Args: + parent (Iterator): The iterator that owns the current page. + items (Sequence[Any]): An iterable (that also defines __len__) of items + from a raw API response. + item_to_value (Callable[Iterator, Any]): Callable to convert an item + from the type in the raw API response into the native object. Will + be called with the iterator and a single item. """ def __init__(self, parent, items, item_to_value): @@ -137,24 +105,16 @@ def __init__(self, parent, items, item_to_value): @property def num_items(self): - """Total items in the page. - - :rtype: int - :returns: The number of items in this page. - """ + """int: Total items in the page.""" return self._num_items @property def remaining(self): - """Remaining items in the page. - - :rtype: int - :returns: The number of items remaining in this page. - """ + """int: Remaining items in the page.""" return self._remaining def __iter__(self): - """The :class:`Page` is an iterator.""" + """The :class:`Page` is an iterator of items.""" return self def next(self): @@ -170,26 +130,28 @@ def next(self): __next__ = next -class Iterator(object): - """A generic class for iterating through API list responses. - - :type client: :class:`~google.cloud.client.Client` - :param client: The client used to identify the application. +def _item_to_value_identity(iterator, item): + """An item to value transformer that returns the item un-changed.""" + # pylint: disable=unused-argument + # We are conforming to the interface defined by Iterator. + return item - :type item_to_value: callable - :param item_to_value: Callable to convert an item from the type in the - raw API response into the native object. - Assumed signature takes an :class:`Iterator` and a - raw API response with a single item. - :type page_token: str - :param page_token: (Optional) A token identifying a page in a result set. +@six.add_metaclass(abc.ABCMeta) +class Iterator(object): + """A generic class for iterating through API list responses. - :type max_results: int - :param max_results: (Optional) The maximum number of results to fetch. + Args: + client(google.cloud.client.Client): The API client. + item_to_value (Callable[Iterator, Any]): Callable to convert an item + from the type in the raw API response into the native object. Will + be called with the iterator and a single item. + page_token (str): A token identifying a page in a result set to start + fetching results from. + max_results (int): The maximum number of results to fetch. """ - def __init__(self, client, item_to_value, + def __init__(self, client, item_to_value=_item_to_value_identity, page_token=None, max_results=None): self._started = False self.client = client @@ -204,9 +166,11 @@ def __init__(self, client, item_to_value, def pages(self): """Iterator of pages in the response. - :rtype: :class:`~types.GeneratorType` - :returns: A generator of :class:`Page` instances. - :raises ValueError: If the iterator has already been started. + returns: + types.GeneratorType[Page]: A generator of :class:`Page` instances. + + raises: + ValueError: If the iterator has already been started. """ if self._started: raise ValueError('Iterator has already started', self) @@ -223,9 +187,11 @@ def _items_iter(self): def __iter__(self): """Iterator for each item returned. - :rtype: :class:`~types.GeneratorType` - :returns: A generator of items from the API. - :raises ValueError: If the iterator has already been started. + Returns: + types.GeneratorType[Any]: A generator of items from the API. + + Raises: + ValueError: If the iterator has already been started. """ if self._started: raise ValueError('Iterator has already started', self) @@ -235,15 +201,14 @@ def __iter__(self): def _page_iter(self, increment): """Generator of pages of API responses. - :type increment: bool - :param increment: Flag indicating if the total number of results - should be incremented on each page. This is useful - since a page iterator will want to increment by - results per page while an items iterator will want - to increment per item. + Args: + increment (bool): Flag indicating if the total number of results + should be incremented on each page. This is useful since a page + iterator will want to increment by results per page while an + items iterator will want to increment per item. - :rtype: :class:`Page` - :returns: pages + Yields: + Page: each page of items from the API. """ page = self._next_page() while page is not None: @@ -253,70 +218,82 @@ def _page_iter(self, increment): yield page page = self._next_page() - @staticmethod - def _next_page(): + @abc.abstractmethod + def _next_page(self): """Get the next page in the iterator. This does nothing and is intended to be over-ridden by subclasses to return the next :class:`Page`. - :raises NotImplementedError: Always. + Raises: + NotImplementedError: Always, this method is abstract. """ raise NotImplementedError -class HTTPIterator(Iterator): - """A generic class for iterating through Cloud JSON APIs list responses. - - :type client: :class:`~google.cloud.client.Client` - :param client: The client used to identify the application. - - :type path: str - :param path: The path to query for the list of items. - - :type item_to_value: callable - :param item_to_value: Callable to convert an item from JSON - into the native object. Assumed signature - takes an :class:`Iterator` and a dictionary - holding a single item. - - :type items_key: str - :param items_key: (Optional) The key used to grab retrieved items from an - API response. Defaults to :data:`DEFAULT_ITEMS_KEY`. +def _do_nothing_page_start(iterator, page, response): + """Helper to provide custom behavior after a :class:`Page` is started. - :type page_token: str - :param page_token: (Optional) A token identifying a page in a result set. + This is a do-nothing stand-in as the default value. - :type max_results: int - :param max_results: (Optional) The maximum number of results to fetch. + Args: + iterator (Iterator): An iterator that holds some request info. + page (Page): The page that was just created. + response (Any): The API response for a page. + """ + # pylint: disable=unused-argument + pass - :type extra_params: dict - :param extra_params: (Optional) Extra query string parameters for the - API call. - :type page_start: callable - :param page_start: (Optional) Callable to provide any special behavior - after a new page has been created. Assumed signature - takes the :class:`Iterator` that started the page, - the :class:`Page` that was started and the dictionary - containing the page response. +class HTTPIterator(Iterator): + """A generic class for iterating through HTTP/JSON API list responses. + + To make an iterator work, you'll need to provide a way to convert a JSON + item returned from the API into the object of your choice (via + ``item_to_value``). You also may need to specify a custom ``items_key`` so + that a given response (containing a page of results) can be parsed into an + iterable page of the actual objects you want. + + Args: + client (google.cloud.client.Client): The API client. + api_request (Callable): The function to use to make API requests. + Generally, this will be + :meth:`google.cloud._http.JSONConnection.api_request`. + path (str): The method path to query for the list of items. + item_to_value (Callable[Iterator, Any]): Callable to convert an item + from the type in the JSON response into a native object. Will + be called with the iterator and a single item. + items_key (str): The key in the API response where the list of items + can be found. + page_token (str): A token identifying a page in a result set to start + fetching results from. + max_results (int): The maximum number of results to fetch. + extra_params (dict): Extra query string parameters for the + API call. + page_start (Callable[Iterator, Page, dict]): Callable to provide any + special behavior after a new page has been created. Assumed + signature takes the :class:`Iterator` that started the page, + the :class:`Page` that was started and the dictionary containing + the page response. .. autoattribute:: pages """ + _DEFAULT_ITEMS_KEY = 'items' _PAGE_TOKEN = 'pageToken' _MAX_RESULTS = 'maxResults' _NEXT_TOKEN = 'nextPageToken' _RESERVED_PARAMS = frozenset([_PAGE_TOKEN, _MAX_RESULTS]) _HTTP_METHOD = 'GET' - def __init__(self, client, path, item_to_value, - items_key=DEFAULT_ITEMS_KEY, + def __init__(self, client, api_request, path, item_to_value, + items_key=_DEFAULT_ITEMS_KEY, page_token=None, max_results=None, extra_params=None, page_start=_do_nothing_page_start): super(HTTPIterator, self).__init__( client, item_to_value, page_token=page_token, max_results=max_results) + self.api_request = api_request self.path = path self._items_key = items_key self.extra_params = extra_params @@ -329,7 +306,8 @@ def __init__(self, client, path, item_to_value, def _verify_params(self): """Verifies the parameters don't use any reserved parameter. - :raises ValueError: If a reserved parameter is used. + Raises: + ValueError: If a reserved parameter is used. """ reserved_in_use = self._RESERVED_PARAMS.intersection( self.extra_params) @@ -340,9 +318,9 @@ def _verify_params(self): def _next_page(self): """Get the next page in the iterator. - :rtype: :class:`Page` - :returns: The next page in the iterator (or :data:`None` if - there are no pages left). + Returns: + Optional[Page]: The next page in the iterator or :data:`None` if + there are no pages left. """ if self._has_next_page(): response = self._get_next_page_response() @@ -357,8 +335,8 @@ def _next_page(self): def _has_next_page(self): """Determines whether or not there are more pages with results. - :rtype: bool - :returns: Whether the iterator has more pages. + Returns: + bool: Whether the iterator has more pages. """ if self.page_number == 0: return True @@ -372,8 +350,8 @@ def _has_next_page(self): def _get_query_params(self): """Getter for query parameters for the next request. - :rtype: dict - :returns: A dictionary of query parameters. + Returns: + dict: A dictionary of query parameters. """ result = {} if self.next_page_token is not None: @@ -386,19 +364,20 @@ def _get_query_params(self): def _get_next_page_response(self): """Requests the next page from the path provided. - :rtype: dict - :returns: The parsed JSON response of the next page's contents. + Returns: + dict: The parsed JSON response of the next page's contents. - :raises ValueError: If the HTTP method is not ``GET`` or ``POST``. + Raises: + ValueError: If the HTTP method is not ``GET`` or ``POST``. """ params = self._get_query_params() if self._HTTP_METHOD == 'GET': - return self.client._connection.api_request( + return self.api_request( method=self._HTTP_METHOD, path=self.path, query_params=params) elif self._HTTP_METHOD == 'POST': - return self.client._connection.api_request( + return self.api_request( method=self._HTTP_METHOD, path=self.path, data=params) @@ -406,30 +385,23 @@ def _get_next_page_response(self): raise ValueError('Unexpected HTTP method', self._HTTP_METHOD) -class GAXIterator(Iterator): +class _GAXIterator(Iterator): """A generic class for iterating through Cloud gRPC APIs list responses. - :type client: :class:`~google.cloud.client.Client` - :param client: The client used to identify the application. - - :type page_iter: :class:`~google.gax.PageIterator` - :param page_iter: A GAX page iterator to be wrapped and conform to the - :class:`~google.cloud.iterator.Iterator` surface. - - :type item_to_value: callable - :param item_to_value: Callable to convert an item from a protobuf - into the native object. Assumed signature - takes an :class:`Iterator` and a single item - from the API response as a protobuf. - - :type max_results: int - :param max_results: (Optional) The maximum number of results to fetch. + Any: + client (google.cloud.client.Client): The API client. + page_iter (google.gax.PageIterator): A GAX page iterator to be wrapped + to conform to the :class:`Iterator` interface. + item_to_value (Callable[Iterator, Any]): Callable to convert an item + from the the protobuf response into a native object. Will + be called with the iterator and a single item. + max_results (int): The maximum number of results to fetch. .. autoattribute:: pages """ def __init__(self, client, page_iter, item_to_value, max_results=None): - super(GAXIterator, self).__init__( + super(_GAXIterator, self).__init__( client, item_to_value, page_token=page_iter.page_token, max_results=max_results) self._gax_page_iter = page_iter @@ -440,9 +412,9 @@ def _next_page(self): Wraps the response from the :class:`~google.gax.PageIterator` in a :class:`Page` instance and captures some state at each page. - :rtype: :class:`Page` - :returns: The next page in the iterator (or :data:`None` if - there are no pages left). + Returns: + Optional[Page]: The next page in the iterator or :data:`None` if + there are no pages left. """ try: items = six.next(self._gax_page_iter) diff --git a/core/tests/unit/api_core/test_page_iterator.py b/core/tests/unit/api_core/test_page_iterator.py new file mode 100644 index 000000000000..82466579e37b --- /dev/null +++ b/core/tests/unit/api_core/test_page_iterator.py @@ -0,0 +1,461 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import types + +import mock +import pytest +import six + +from google.api.core import page_iterator + + +def test__do_nothing_page_start(): + assert page_iterator._do_nothing_page_start(None, None, None) is None + + +class TestPage(object): + + def test_constructor(self): + parent = mock.sentinel.parent + item_to_value = mock.sentinel.item_to_value + + page = page_iterator.Page(parent, (1, 2, 3), item_to_value) + + assert page.num_items == 3 + assert page.remaining == 3 + assert page._parent is parent + assert page._item_to_value is item_to_value + + def test___iter__(self): + page = page_iterator.Page(None, (), None) + assert iter(page) is page + + def test_iterator_calls_parent_item_to_value(self): + parent = mock.sentinel.parent + + item_to_value = mock.Mock( + side_effect=lambda iterator, value: value, spec=['__call__']) + + page = page_iterator.Page(parent, (10, 11, 12), item_to_value) + page._remaining = 100 + + assert item_to_value.call_count == 0 + assert page.remaining == 100 + + assert six.next(page) == 10 + assert item_to_value.call_count == 1 + item_to_value.assert_called_with(parent, 10) + assert page.remaining == 99 + + assert six.next(page) == 11 + assert item_to_value.call_count == 2 + item_to_value.assert_called_with(parent, 11) + assert page.remaining == 98 + + assert six.next(page) == 12 + assert item_to_value.call_count == 3 + item_to_value.assert_called_with(parent, 12) + assert page.remaining == 97 + + +class PageIteratorImpl(page_iterator.Iterator): + def _next_page(self): + return mock.create_autospec(page_iterator.Page, instance=True) + + +class TestIterator(object): + + def test_constructor(self): + client = mock.sentinel.client + item_to_value = mock.sentinel.item_to_value + token = 'ab13nceor03' + max_results = 1337 + + iterator = PageIteratorImpl( + client, item_to_value, page_token=token, max_results=max_results) + + assert not iterator._started + assert iterator.client is client + assert iterator._item_to_value == item_to_value + assert iterator.max_results == max_results + # Changing attributes. + assert iterator.page_number == 0 + assert iterator.next_page_token == token + assert iterator.num_results == 0 + + def test_pages_property_starts(self): + iterator = PageIteratorImpl(None, None) + + assert not iterator._started + + assert isinstance(iterator.pages, types.GeneratorType) + + assert iterator._started + + def test_pages_property_restart(self): + iterator = PageIteratorImpl(None, None) + + assert iterator.pages + + # Make sure we cannot restart. + with pytest.raises(ValueError): + assert iterator.pages + + def test__page_iter_increment(self): + iterator = PageIteratorImpl(None, None) + page = page_iterator.Page( + iterator, ('item',), page_iterator._item_to_value_identity) + iterator._next_page = mock.Mock(side_effect=[page, None]) + + assert iterator.num_results == 0 + + page_iter = iterator._page_iter(increment=True) + next(page_iter) + + assert iterator.num_results == 1 + + def test__page_iter_no_increment(self): + iterator = PageIteratorImpl(None, None) + + assert iterator.num_results == 0 + + page_iter = iterator._page_iter(increment=False) + next(page_iter) + + # results should still be 0 after fetching a page. + assert iterator.num_results == 0 + + def test__items_iter(self): + # Items to be returned. + item1 = 17 + item2 = 100 + item3 = 211 + + # Make pages from mock responses + parent = mock.sentinel.parent + page1 = page_iterator.Page( + parent, (item1, item2), page_iterator._item_to_value_identity) + page2 = page_iterator.Page( + parent, (item3,), page_iterator._item_to_value_identity) + + iterator = PageIteratorImpl(None, None) + iterator._next_page = mock.Mock(side_effect=[page1, page2, None]) + + items_iter = iterator._items_iter() + + assert isinstance(items_iter, types.GeneratorType) + + # Consume items and check the state of the iterator. + assert iterator.num_results == 0 + + assert six.next(items_iter) == item1 + assert iterator.num_results == 1 + + assert six.next(items_iter) == item2 + assert iterator.num_results == 2 + + assert six.next(items_iter) == item3 + assert iterator.num_results == 3 + + with pytest.raises(StopIteration): + six.next(items_iter) + + def test___iter__(self): + iterator = PageIteratorImpl(None, None) + iterator._next_page = mock.Mock(side_effect=[(1, 2), (3,), None]) + + assert not iterator._started + + result = list(iterator) + + assert result == [1, 2, 3] + assert iterator._started + + def test___iter__restart(self): + iterator = PageIteratorImpl(None, None) + + iter(iterator) + + # Make sure we cannot restart. + with pytest.raises(ValueError): + iter(iterator) + + def test___iter___restart_after_page(self): + iterator = PageIteratorImpl(None, None) + + assert iterator.pages + + # Make sure we cannot restart after starting the page iterator + with pytest.raises(ValueError): + iter(iterator) + + +class TestHTTPIterator(object): + + def test_constructor(self): + client = mock.sentinel.client + path = '/foo' + iterator = page_iterator.HTTPIterator( + client, mock.sentinel.api_request, + path, mock.sentinel.item_to_value) + + assert not iterator._started + assert iterator.client is client + assert iterator.path == path + assert iterator._item_to_value is mock.sentinel.item_to_value + assert iterator._items_key == 'items' + assert iterator.max_results is None + assert iterator.extra_params == {} + assert iterator._page_start == page_iterator._do_nothing_page_start + # Changing attributes. + assert iterator.page_number == 0 + assert iterator.next_page_token is None + assert iterator.num_results == 0 + + def test_constructor_w_extra_param_collision(self): + extra_params = {'pageToken': 'val'} + + with pytest.raises(ValueError): + page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value, + extra_params=extra_params) + + def test_iterate(self): + path = '/foo' + item1 = {'name': '1'} + item2 = {'name': '2'} + api_request = mock.Mock(return_value={'items': [item1, item2]}) + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, api_request, path=path, + item_to_value=page_iterator._item_to_value_identity) + + assert iterator.num_results == 0 + + items_iter = iter(iterator) + + val1 = six.next(items_iter) + assert val1 == item1 + assert iterator.num_results == 1 + + val2 = six.next(items_iter) + assert val2 == item2 + assert iterator.num_results == 2 + + with pytest.raises(StopIteration): + six.next(items_iter) + + api_request.assert_called_once_with( + method='GET', path=path, query_params={}) + + def test__has_next_page_new(self): + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value) + + # The iterator should *always* indicate that it has a next page + # when created so that it can fetch the initial page. + assert iterator._has_next_page() + + def test__has_next_page_without_token(self): + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value) + + iterator.page_number = 1 + + # The iterator should not indicate that it has a new page if the + # initial page has been requested and there's no page token. + assert not iterator._has_next_page() + + def test__has_next_page_w_number_w_token(self): + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value) + + iterator.page_number = 1 + iterator.next_page_token = mock.sentinel.token + + # The iterator should indicate that it has a new page if the + # initial page has been requested and there's is a page token. + assert iterator._has_next_page() + + def test__has_next_page_w_max_results_not_done(self): + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value, + max_results=3, + page_token=mock.sentinel.token) + + iterator.page_number = 1 + + # The iterator should indicate that it has a new page if there + # is a page token and it has not consumed more than max_results. + assert iterator.num_results < iterator.max_results + assert iterator._has_next_page() + + def test__has_next_page_w_max_results_done(self): + + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value, + max_results=3, + page_token=mock.sentinel.token) + + iterator.page_number = 1 + iterator.num_results = 3 + + # The iterator should not indicate that it has a new page if there + # if it has consumed more than max_results. + assert iterator.num_results == iterator.max_results + assert not iterator._has_next_page() + + def test__get_query_params_no_token(self): + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value) + + assert iterator._get_query_params() == {} + + def test__get_query_params_w_token(self): + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value) + iterator.next_page_token = 'token' + + assert iterator._get_query_params() == { + 'pageToken': iterator.next_page_token} + + def test__get_query_params_w_max_results(self): + max_results = 3 + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value, + max_results=max_results) + + iterator.num_results = 1 + local_max = max_results - iterator.num_results + + assert iterator._get_query_params() == { + 'maxResults': local_max} + + def test__get_query_params_extra_params(self): + extra_params = {'key': 'val'} + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value, + extra_params=extra_params) + + assert iterator._get_query_params() == extra_params + + def test__get_next_page_response_with_post(self): + path = '/foo' + page_response = {'items': ['one', 'two']} + api_request = mock.Mock(return_value=page_response) + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, api_request, path=path, + item_to_value=page_iterator._item_to_value_identity) + iterator._HTTP_METHOD = 'POST' + + response = iterator._get_next_page_response() + + assert response == page_response + + api_request.assert_called_once_with( + method='POST', path=path, data={}) + + def test__get_next_page_bad_http_method(self): + iterator = page_iterator.HTTPIterator( + mock.sentinel.client, + mock.sentinel.api_request, + mock.sentinel.path, + mock.sentinel.item_to_value) + iterator._HTTP_METHOD = 'NOT-A-VERB' + + with pytest.raises(ValueError): + iterator._get_next_page_response() + + +class GAXPageIterator(object): + """Fake object that matches gax.PageIterator""" + def __init__(self, pages, page_token=None): + self._pages = iter(pages) + self.page_token = page_token + + def next(self): + return six.next(self._pages) + + __next__ = next + + +class TestGAXIterator(object): + + def test_constructor(self): + client = mock.sentinel.client + token = 'zzzyy78kl' + page_iter = GAXPageIterator((), page_token=token) + item_to_value = page_iterator._item_to_value_identity + max_results = 1337 + iterator = page_iterator._GAXIterator( + client, page_iter, item_to_value, max_results=max_results) + + assert not iterator._started + assert iterator.client is client + assert iterator._item_to_value is item_to_value + assert iterator.max_results == max_results + assert iterator._gax_page_iter is page_iter + # Changing attributes. + assert iterator.page_number == 0 + assert iterator.next_page_token == token + assert iterator.num_results == 0 + + def test__next_page(self): + page_items = (29, 31) + page_token = '2sde98ds2s0hh' + page_iter = GAXPageIterator([page_items], page_token=page_token) + iterator = page_iterator._GAXIterator( + mock.sentinel.client, + page_iter, + page_iterator._item_to_value_identity) + + page = iterator._next_page() + + assert iterator.next_page_token == page_token + assert isinstance(page, page_iterator.Page) + assert list(page) == list(page_items) + + next_page = iterator._next_page() + + assert next_page is None diff --git a/core/tests/unit/test_iterator.py b/core/tests/unit/test_iterator.py deleted file mode 100644 index a7d9e4f0924d..000000000000 --- a/core/tests/unit/test_iterator.py +++ /dev/null @@ -1,605 +0,0 @@ -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - - -class Test__do_nothing_page_start(unittest.TestCase): - - def _call_fut(self, iterator, page, response): - from google.cloud.iterator import _do_nothing_page_start - - return _do_nothing_page_start(iterator, page, response) - - def test_do_nothing(self): - result = self._call_fut(None, None, None) - self.assertIsNone(result) - - -class TestPage(unittest.TestCase): - - @staticmethod - def _get_target_class(): - from google.cloud.iterator import Page - - return Page - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_constructor(self): - parent = object() - item_to_value = object() - page = self._make_one(parent, (1, 2, 3), item_to_value) - self.assertIs(page._parent, parent) - self.assertEqual(page._num_items, 3) - self.assertEqual(page._remaining, 3) - self.assertIs(page._item_to_value, item_to_value) - - def test_num_items_property(self): - page = self._make_one(None, (), None) - num_items = 42 - page._num_items = num_items - self.assertEqual(page.num_items, num_items) - - def test_remaining_property(self): - page = self._make_one(None, (), None) - remaining = 1337 - page._remaining = remaining - self.assertEqual(page.remaining, remaining) - - def test___iter__(self): - page = self._make_one(None, (), None) - self.assertIs(iter(page), page) - - def test_iterator_calls__item_to_value(self): - import six - - class Parent(object): - - calls = 0 - - def item_to_value(self, item): - self.calls += 1 - return item - - parent = Parent() - page = self._make_one(parent, (10, 11, 12), - Parent.item_to_value) - page._remaining = 100 - - self.assertEqual(parent.calls, 0) - self.assertEqual(page.remaining, 100) - self.assertEqual(six.next(page), 10) - self.assertEqual(parent.calls, 1) - self.assertEqual(page.remaining, 99) - self.assertEqual(six.next(page), 11) - self.assertEqual(parent.calls, 2) - self.assertEqual(page.remaining, 98) - self.assertEqual(six.next(page), 12) - self.assertEqual(parent.calls, 3) - self.assertEqual(page.remaining, 97) - - -class TestIterator(unittest.TestCase): - - @staticmethod - def _get_target_class(): - from google.cloud.iterator import Iterator - - return Iterator - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_constructor(self): - connection = _Connection() - client = _Client(connection) - item_to_value = object() - token = 'ab13nceor03' - max_results = 1337 - iterator = self._make_one(client, item_to_value, page_token=token, - max_results=max_results) - - self.assertFalse(iterator._started) - self.assertIs(iterator.client, client) - self.assertIs(iterator._item_to_value, item_to_value) - self.assertEqual(iterator.max_results, max_results) - # Changing attributes. - self.assertEqual(iterator.page_number, 0) - self.assertEqual(iterator.next_page_token, token) - self.assertEqual(iterator.num_results, 0) - - def test_pages_property(self): - iterator = self._make_one(None, None) - self.assertFalse(iterator._started) - mock_iter = object() - incremented = [] - - def page_iter(increment): - incremented.append(increment) - return mock_iter - - iterator._page_iter = page_iter - self.assertIs(iterator.pages, mock_iter) - self.assertEqual(incremented, [True]) - # Check the side-effect. - self.assertTrue(iterator._started) - - def test_pages_property_started(self): - import types - - iterator = self._make_one(None, None) - self.assertIsInstance(iterator.pages, types.GeneratorType) - # Make sure we cannot restart. - with self.assertRaises(ValueError): - getattr(iterator, 'pages') - - def test_pages_property_items_started(self): - import types - - iterator = self._make_one(None, None) - self.assertIsInstance(iter(iterator), types.GeneratorType) - with self.assertRaises(ValueError): - getattr(iterator, 'pages') - - @staticmethod - def _do_nothing(parent, value): - return parent, value - - def test__items_iter(self): - import types - import six - from google.cloud.iterator import Page - - # Items to be returned. - item1 = 17 - item2 = 100 - item3 = 211 - - # Make pages from mock responses - parent = object() - page1 = Page(parent, (item1, item2), self._do_nothing) - page2 = Page(parent, (item3,), self._do_nothing) - - iterator = self._make_one(None, None) - # Fake the page iterator on the object. - incremented = [] - - def page_iter(increment): - incremented.append(increment) - return iter((page1, page2)) - - iterator._page_iter = page_iter - items_iter = iterator._items_iter() - # Make sure it is a generator. - self.assertIsInstance(items_iter, types.GeneratorType) - - # Consume items and check the state of the iterator. - self.assertEqual(iterator.num_results, 0) - self.assertEqual(six.next(items_iter), (parent, item1)) - self.assertEqual(iterator.num_results, 1) - self.assertEqual(six.next(items_iter), (parent, item2)) - self.assertEqual(iterator.num_results, 2) - self.assertEqual(six.next(items_iter), (parent, item3)) - self.assertEqual(iterator.num_results, 3) - with self.assertRaises(StopIteration): - six.next(items_iter) - - # Make sure our page_iter() was called correctly. - self.assertEqual(incremented, [False]) - - def test___iter__(self): - iterator = self._make_one(None, None) - self.assertFalse(iterator._started) - incremented = [] - - def page_iter(increment): - incremented.append(increment) - return iter(()) - - iterator._page_iter = page_iter - self.assertEqual(list(iterator), []) - # Check the side-effect. - self.assertTrue(iterator._started) - - def test___iter___started(self): - import types - - iterator = self._make_one(None, None) - self.assertIsInstance(iter(iterator), types.GeneratorType) - with self.assertRaises(ValueError): - iter(iterator) - - def test___iter___pages_started(self): - import types - - iterator = self._make_one(None, None) - self.assertIsInstance(iterator.pages, types.GeneratorType) - with self.assertRaises(ValueError): - iter(iterator) - - def test__next_page_virtual(self): - iterator = self._make_one(None, None) - with self.assertRaises(NotImplementedError): - iterator._next_page() - - -class TestHTTPIterator(unittest.TestCase): - - @staticmethod - def _get_target_class(): - from google.cloud.iterator import HTTPIterator - - return HTTPIterator - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_constructor(self): - from google.cloud.iterator import _do_nothing_page_start - - connection = _Connection() - client = _Client(connection) - path = '/foo' - iterator = self._make_one(client, path, None) - self.assertFalse(iterator._started) - self.assertIs(iterator.client, client) - self.assertEqual(iterator.path, path) - self.assertIsNone(iterator._item_to_value) - self.assertEqual(iterator._items_key, 'items') - self.assertIsNone(iterator.max_results) - self.assertEqual(iterator.extra_params, {}) - self.assertIs(iterator._page_start, _do_nothing_page_start) - # Changing attributes. - self.assertEqual(iterator.page_number, 0) - self.assertIsNone(iterator.next_page_token) - self.assertEqual(iterator.num_results, 0) - - def test_constructor_w_extra_param_collision(self): - connection = _Connection() - client = _Client(connection) - path = '/foo' - extra_params = {'pageToken': 'val'} - with self.assertRaises(ValueError): - self._make_one(client, path, None, extra_params=extra_params) - - def test_pages_iter_empty_then_another(self): - import six - from google.cloud._testing import _Monkey - from google.cloud import iterator as MUT - - items_key = 'its-key' - iterator = self._make_one(None, None, None, items_key=items_key) - # Fake the next page class. - fake_page = MUT.Page(None, (), None) - page_args = [] - - def dummy_response(): - return {} - - def dummy_page_class(*args): - page_args.append(args) - return fake_page - - iterator._get_next_page_response = dummy_response - pages_iter = iterator.pages - with _Monkey(MUT, Page=dummy_page_class): - page = six.next(pages_iter) - self.assertIs(page, fake_page) - self.assertEqual( - page_args, [(iterator, (), iterator._item_to_value)]) - - def test_iterate(self): - import six - - path = '/foo' - key1 = 'key1' - key2 = 'key2' - item1, item2 = object(), object() - ITEMS = {key1: item1, key2: item2} - - def item_to_value(iterator, item): # pylint: disable=unused-argument - return ITEMS[item['name']] - - connection = _Connection( - {'items': [{'name': key1}, {'name': key2}]}) - client = _Client(connection) - iterator = self._make_one(client, path=path, - item_to_value=item_to_value) - self.assertEqual(iterator.num_results, 0) - - items_iter = iter(iterator) - val1 = six.next(items_iter) - self.assertEqual(val1, item1) - self.assertEqual(iterator.num_results, 1) - - val2 = six.next(items_iter) - self.assertEqual(val2, item2) - self.assertEqual(iterator.num_results, 2) - - with self.assertRaises(StopIteration): - six.next(items_iter) - - kw, = connection._requested - self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['path'], path) - self.assertEqual(kw['query_params'], {}) - - def test__has_next_page_new(self): - connection = _Connection() - client = _Client(connection) - path = '/foo' - iterator = self._make_one(client, path, None) - self.assertTrue(iterator._has_next_page()) - - def test__has_next_page_w_number_no_token(self): - connection = _Connection() - client = _Client(connection) - path = '/foo' - iterator = self._make_one(client, path, None) - iterator.page_number = 1 - self.assertFalse(iterator._has_next_page()) - - def test__has_next_page_w_number_w_token(self): - connection = _Connection() - client = _Client(connection) - path = '/foo' - token = 'token' - iterator = self._make_one(client, path, None) - iterator.page_number = 1 - iterator.next_page_token = token - self.assertTrue(iterator._has_next_page()) - - def test__has_next_page_w_max_results_not_done(self): - iterator = self._make_one(None, None, None, max_results=3, - page_token='definitely-not-none') - iterator.page_number = 1 - self.assertLess(iterator.num_results, iterator.max_results) - self.assertTrue(iterator._has_next_page()) - - def test__has_next_page_w_max_results_done(self): - iterator = self._make_one(None, None, None, max_results=3) - iterator.page_number = 1 - iterator.num_results = iterator.max_results - self.assertFalse(iterator._has_next_page()) - - def test__get_query_params_no_token(self): - connection = _Connection() - client = _Client(connection) - path = '/foo' - iterator = self._make_one(client, path, None) - self.assertEqual(iterator._get_query_params(), {}) - - def test__get_query_params_w_token(self): - connection = _Connection() - client = _Client(connection) - path = '/foo' - token = 'token' - iterator = self._make_one(client, path, None) - iterator.next_page_token = token - self.assertEqual(iterator._get_query_params(), - {'pageToken': token}) - - def test__get_query_params_w_max_results(self): - connection = _Connection() - client = _Client(connection) - path = '/foo' - max_results = 3 - iterator = self._make_one(client, path, None, - max_results=max_results) - iterator.num_results = 1 - local_max = max_results - iterator.num_results - self.assertEqual(iterator._get_query_params(), - {'maxResults': local_max}) - - def test__get_query_params_extra_params(self): - connection = _Connection() - client = _Client(connection) - path = '/foo' - extra_params = {'key': 'val'} - iterator = self._make_one(client, path, None, - extra_params=extra_params) - self.assertEqual(iterator._get_query_params(), extra_params) - - def test__get_query_params_w_token_and_extra_params(self): - connection = _Connection() - client = _Client(connection) - path = '/foo' - token = 'token' - extra_params = {'key': 'val'} - iterator = self._make_one(client, path, None, - extra_params=extra_params) - iterator.next_page_token = token - - expected_query = extra_params.copy() - expected_query.update({'pageToken': token}) - self.assertEqual(iterator._get_query_params(), expected_query) - - def test__get_next_page_response_new_no_token_in_response(self): - path = '/foo' - token = 'token' - key1 = 'key1' - key2 = 'key2' - connection = _Connection({'items': [{'name': key1}, {'name': key2}], - 'nextPageToken': token}) - client = _Client(connection) - iterator = self._make_one(client, path, None) - response = iterator._get_next_page_response() - self.assertEqual(response['items'], [{'name': key1}, {'name': key2}]) - kw, = connection._requested - self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['path'], path) - self.assertEqual(kw['query_params'], {}) - - def test__get_next_page_response_with_post(self): - path = '/foo' - returned = {'green': 'eggs', 'ham': 55} - connection = _Connection(returned) - client = _Client(connection) - iterator = self._make_one(client, path, None) - iterator._HTTP_METHOD = 'POST' - response = iterator._get_next_page_response() - self.assertEqual(response, returned) - - self.assertEqual(len(connection._requested), 1) - called_kwargs = connection._requested[0] - self.assertEqual(called_kwargs, { - 'method': iterator._HTTP_METHOD, - 'path': path, - 'data': {}, - }) - - def test__get_next_page_bad_http_method(self): - path = '/foo' - client = _Client(None) - iterator = self._make_one(client, path, None) - iterator._HTTP_METHOD = 'NOT-A-VERB' - with self.assertRaises(ValueError): - iterator._get_next_page_response() - - -class TestGAXIterator(unittest.TestCase): - - @staticmethod - def _get_target_class(): - from google.cloud.iterator import GAXIterator - - return GAXIterator - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_constructor(self): - client = _Client(None) - token = 'zzzyy78kl' - page_iter = SimpleIter(token) - item_to_value = object() - max_results = 1337 - iterator = self._make_one(client, page_iter, item_to_value, - max_results=max_results) - - self.assertFalse(iterator._started) - self.assertIs(iterator.client, client) - self.assertIs(iterator._item_to_value, item_to_value) - self.assertEqual(iterator.max_results, max_results) - self.assertIs(iterator._gax_page_iter, page_iter) - # Changing attributes. - self.assertEqual(iterator.page_number, 0) - self.assertEqual(iterator.next_page_token, token) - self.assertEqual(iterator.num_results, 0) - - @staticmethod - def _do_nothing(parent, value): - return parent, value - - def test__next_page(self): - from google.cloud._testing import _GAXPageIterator - from google.cloud.iterator import Page - - # Make a mock ``google.gax.PageIterator`` - page_items = (29, 31) # Items for just one page. - page_token = '2sde98ds2s0hh' - page_iter = _GAXPageIterator(page_items, page_token=page_token) - # Wrap the GAX iterator. - iterator = self._make_one(None, page_iter, self._do_nothing) - - page = iterator._next_page() - # First check the page token. - self.assertEqual(iterator.next_page_token, page_token) - # Then check the page. - self.assertIsInstance(page, Page) - # _do_nothing will throw the iterator in front. - expected = zip((iterator, iterator), page_items) - self.assertEqual(list(page), list(expected)) - - def test__next_page_empty(self): - from google.cloud._testing import _GAXPageIterator - - # Make a mock ``google.gax.PageIterator`` - page_iter = _GAXPageIterator() - # Wrap the GAX iterator. - iterator = self._make_one(None, page_iter, self._do_nothing) - - page = iterator._next_page() - self.assertIsNone(page) - self.assertIsNone(iterator.next_page_token) - - def test_iterate(self): - import six - from google.cloud._testing import _GAXPageIterator - - item1 = object() - item2 = object() - item3 = object() - token1 = 'smkdme30e2e32r' - token2 = '39cm9csl123dck' - - # Make a mock ``google.gax.PageIterator`` - page1 = (item1,) - page2 = (item2, item3) - page_iter = _GAXPageIterator(page1, page2, page_token=token1) - iterator = self._make_one(None, page_iter, self._do_nothing) - - self.assertEqual(iterator.num_results, 0) - - items_iter = iter(iterator) - val1 = six.next(items_iter) - self.assertEqual(val1, (iterator, item1)) - self.assertEqual(iterator.num_results, 1) - self.assertEqual(iterator.next_page_token, token1) - - # Before grabbing the next page, hot-swap the token - # on the ``page_iter``. - page_iter.page_token = token2 - - # Grab the next item (which will cause the next page). - val2 = six.next(items_iter) - self.assertEqual(val2, (iterator, item2)) - self.assertEqual(iterator.num_results, 2) - self.assertEqual(iterator.next_page_token, token2) - - # Grab the final item from the final / current page. - val3 = six.next(items_iter) - self.assertEqual(val3, (iterator, item3)) - self.assertEqual(iterator.num_results, 3) - # Make sure the token did not change. - self.assertEqual(iterator.next_page_token, token2) - - with self.assertRaises(StopIteration): - six.next(items_iter) - - -class _Connection(object): - - def __init__(self, *responses): - self._responses = responses - self._requested = [] - - def api_request(self, **kw): - self._requested.append(kw) - response, self._responses = self._responses[0], self._responses[1:] - return response - - -class _Client(object): - - def __init__(self, connection): - self._connection = connection - - -class SimpleIter(object): - - def __init__(self, page_token=None): - self.page_token = page_token diff --git a/datastore/google/cloud/datastore/query.py b/datastore/google/cloud/datastore/query.py index 2ab65064f85e..d0c9cea9f711 100644 --- a/datastore/google/cloud/datastore/query.py +++ b/datastore/google/cloud/datastore/query.py @@ -16,9 +16,8 @@ import base64 +from google.api.core import page_iterator from google.cloud._helpers import _ensure_tuple_or_list -from google.cloud.iterator import Iterator as BaseIterator -from google.cloud.iterator import Page from google.cloud.proto.datastore.v1 import datastore_pb2 as _datastore_pb2 from google.cloud.proto.datastore.v1 import entity_pb2 as _entity_pb2 @@ -373,7 +372,7 @@ def fetch(self, limit=None, offset=0, start_cursor=None, end_cursor=None, start_cursor=start_cursor, end_cursor=end_cursor) -class Iterator(BaseIterator): +class Iterator(page_iterator.Iterator): """Represent the state of a given execution of a Query. :type query: :class:`~google.cloud.datastore.query.Query` @@ -499,7 +498,7 @@ def _next_page(self): query=query_pb, ) entity_pbs = self._process_query_results(response_pb) - return Page(self, entity_pbs, self._item_to_value) + return page_iterator.Page(self, entity_pbs, self._item_to_value) def _pb_from_query(query): @@ -571,7 +570,7 @@ def _pb_from_query(query): def _item_to_entity(iterator, entity_pb): """Convert a raw protobuf entity to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type entity_pb: diff --git a/datastore/tests/unit/test_query.py b/datastore/tests/unit/test_query.py index 26c1b6cc0831..d8d08430dab9 100644 --- a/datastore/tests/unit/test_query.py +++ b/datastore/tests/unit/test_query.py @@ -488,7 +488,7 @@ def test__process_query_results_bad_enum(self): iterator._process_query_results(response_pb) def _next_page_helper(self, txn_id=None): - from google.cloud.iterator import Page + from google.api.core import page_iterator from google.cloud.proto.datastore.v1 import datastore_pb2 from google.cloud.proto.datastore.v1 import entity_pb2 from google.cloud.proto.datastore.v1 import query_pb2 @@ -509,7 +509,7 @@ def _next_page_helper(self, txn_id=None): iterator = self._make_one(query, client) page = iterator._next_page() - self.assertIsInstance(page, Page) + self.assertIsInstance(page, page_iterator.Page) self.assertIs(page._parent, iterator) partition_id = entity_pb2.PartitionId(project_id=project) diff --git a/dns/google/cloud/dns/client.py b/dns/google/cloud/dns/client.py index 4025f7e9eb68..c800d5897560 100644 --- a/dns/google/cloud/dns/client.py +++ b/dns/google/cloud/dns/client.py @@ -14,12 +14,11 @@ """Client for interacting with the Google Cloud DNS API.""" - +from google.api.core import page_iterator from google.cloud.client import ClientWithProject from google.cloud.dns._http import Connection from google.cloud.dns.zone import ManagedZone -from google.cloud.iterator import HTTPIterator class Client(ClientWithProject): @@ -86,14 +85,18 @@ def list_zones(self, max_results=None, page_token=None): not passed, the API will return the first page of zones. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.dns.zone.ManagedZone` belonging to this project. """ path = '/projects/%s/managedZones' % (self.project,) - return HTTPIterator( - client=self, path=path, item_to_value=_item_to_zone, - items_key='managedZones', page_token=page_token, + return page_iterator.HTTPIterator( + client=self, + api_request=self._connection.api_request, + path=path, + item_to_value=_item_to_zone, + items_key='managedZones', + page_token=page_token, max_results=max_results) def zone(self, name, dns_name=None, description=None): @@ -122,7 +125,7 @@ def zone(self, name, dns_name=None, description=None): def _item_to_zone(iterator, resource): """Convert a JSON managed zone to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that has retrieved the item. :type resource: dict diff --git a/dns/google/cloud/dns/zone.py b/dns/google/cloud/dns/zone.py index 4c278e903862..db9aa68df071 100644 --- a/dns/google/cloud/dns/zone.py +++ b/dns/google/cloud/dns/zone.py @@ -16,11 +16,11 @@ import six +from google.api.core import page_iterator from google.cloud._helpers import _rfc3339_to_datetime from google.cloud.exceptions import NotFound from google.cloud.dns.changes import Changes from google.cloud.dns.resource_record_set import ResourceRecordSet -from google.cloud.iterator import HTTPIterator class ManagedZone(object): @@ -340,17 +340,21 @@ def list_resource_record_sets(self, max_results=None, page_token=None, (Optional) the client to use. If not passed, falls back to the ``client`` stored on the current zone. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~.resource_record_set.ResourceRecordSet` belonging to this zone. """ client = self._require_client(client) path = '/projects/%s/managedZones/%s/rrsets' % ( self.project, self.name) - iterator = HTTPIterator( - client=client, path=path, - item_to_value=_item_to_resource_record_set, items_key='rrsets', - page_token=page_token, max_results=max_results) + iterator = page_iterator.HTTPIterator( + client=client, + api_request=client._connection.api_request, + path=path, + item_to_value=_item_to_resource_record_set, + items_key='rrsets', + page_token=page_token, + max_results=max_results) iterator.zone = self return iterator @@ -374,16 +378,20 @@ def list_changes(self, max_results=None, page_token=None, client=None): (Optional) the client to use. If not passed, falls back to the ``client`` stored on the current zone. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~.changes.Changes` belonging to this zone. """ client = self._require_client(client) path = '/projects/%s/managedZones/%s/changes' % ( self.project, self.name) - iterator = HTTPIterator( - client=client, path=path, item_to_value=_item_to_changes, - items_key='changes', page_token=page_token, + iterator = page_iterator.HTTPIterator( + client=client, + api_request=client._connection.api_request, + path=path, + item_to_value=_item_to_changes, + items_key='changes', + page_token=page_token, max_results=max_results) iterator.zone = self return iterator @@ -392,7 +400,7 @@ def list_changes(self, max_results=None, page_token=None, client=None): def _item_to_resource_record_set(iterator, resource): """Convert a JSON resource record set value to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that has retrieved the item. :type resource: dict @@ -407,7 +415,7 @@ def _item_to_resource_record_set(iterator, resource): def _item_to_changes(iterator, resource): """Convert a JSON "changes" value to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that has retrieved the item. :type resource: dict diff --git a/docs/core/index.rst b/docs/core/index.rst index 58985beec8f1..964609401655 100644 --- a/docs/core/index.rst +++ b/docs/core/index.rst @@ -4,7 +4,6 @@ Core .. toctree:: config auth - iterators operation-api modules diff --git a/docs/core/iterators.rst b/docs/core/iterators.rst deleted file mode 100644 index b53a41fe40fe..000000000000 --- a/docs/core/iterators.rst +++ /dev/null @@ -1,6 +0,0 @@ -Iterators -~~~~~~~~~ - -.. automodule:: google.cloud.iterator - :members: - :show-inheritance: diff --git a/logging/google/cloud/logging/_gax.py b/logging/google/cloud/logging/_gax.py index 3fb648d98f7f..bfea5df022ad 100644 --- a/logging/google/cloud/logging/_gax.py +++ b/logging/google/cloud/logging/_gax.py @@ -16,6 +16,7 @@ import functools +from google.api.core import page_iterator from google.cloud.gapic.logging.v2.config_service_v2_client import ( ConfigServiceV2Client) from google.cloud.gapic.logging.v2.logging_service_v2_client import ( @@ -37,7 +38,6 @@ from google.cloud._http import DEFAULT_USER_AGENT from google.cloud.exceptions import Conflict from google.cloud.exceptions import NotFound -from google.cloud.iterator import GAXIterator from google.cloud.logging import __version__ from google.cloud.logging._helpers import entry_from_resource from google.cloud.logging.sink import Sink @@ -84,7 +84,7 @@ def list_entries(self, projects, filter_='', order_by='', passed, the API will return the first page of entries. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.logging.entries._BaseEntry` accessible to the current API. """ @@ -101,7 +101,8 @@ def list_entries(self, projects, filter_='', order_by='', loggers = {} item_to_value = functools.partial( _item_to_entry, loggers=loggers) - return GAXIterator(self._client, page_iter, item_to_value) + return page_iterator._GAXIterator( + self._client, page_iter, item_to_value) def write_entries(self, entries, logger_name=None, resource=None, labels=None): @@ -188,7 +189,8 @@ def list_sinks(self, project, page_size=0, page_token=None): path = 'projects/%s' % (project,) page_iter = self._gax_api.list_sinks(path, page_size=page_size, options=options) - return GAXIterator(self._client, page_iter, _item_to_sink) + return page_iterator._GAXIterator( + self._client, page_iter, _item_to_sink) def sink_create(self, project, sink_name, filter_, destination): """API call: create a sink resource. @@ -330,7 +332,7 @@ def list_metrics(self, project, page_size=0, page_token=None): passed, the API will return the first page of metrics. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.logging.metric.Metric` accessible to the current API. @@ -341,7 +343,8 @@ def list_metrics(self, project, page_size=0, page_token=None): path = 'projects/%s' % (project,) page_iter = self._gax_api.list_log_metrics( path, page_size=page_size, options=options) - return GAXIterator(self._client, page_iter, _item_to_metric) + return page_iterator._GAXIterator( + self._client, page_iter, _item_to_metric) def metric_create(self, project, metric_name, filter_, description): """API call: create a metric resource. @@ -507,12 +510,12 @@ def _item_to_entry(iterator, entry_pb, loggers): This method does not have the correct signature to be used as the ``item_to_value`` argument to - :class:`~google.cloud.iterator.Iterator`. It is intended to be + :class:`~google.api.core.page_iterator.Iterator`. It is intended to be patched with a mutable ``loggers`` argument that can be updated on subsequent calls. For an example, see how the method is used above in :meth:`_LoggingAPI.list_entries`. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type entry_pb: :class:`.log_entry_pb2.LogEntry` @@ -534,7 +537,7 @@ def _item_to_entry(iterator, entry_pb, loggers): def _item_to_sink(iterator, log_sink_pb): """Convert a sink protobuf to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type log_sink_pb: @@ -553,7 +556,7 @@ def _item_to_sink(iterator, log_sink_pb): def _item_to_metric(iterator, log_metric_pb): """Convert a metric protobuf to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type log_metric_pb: diff --git a/logging/google/cloud/logging/_http.py b/logging/google/cloud/logging/_http.py index 7ca5c457c25d..45db345fa847 100644 --- a/logging/google/cloud/logging/_http.py +++ b/logging/google/cloud/logging/_http.py @@ -16,8 +16,8 @@ import functools +from google.api.core import page_iterator from google.cloud import _http -from google.cloud.iterator import HTTPIterator from google.cloud.logging import __version__ from google.cloud.logging._helpers import entry_from_resource @@ -93,7 +93,7 @@ def list_entries(self, projects, filter_=None, order_by=None, passed, the API will return the first page of entries. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.logging.entries._BaseEntry` accessible to the current API. """ @@ -115,10 +115,14 @@ def list_entries(self, projects, filter_=None, order_by=None, loggers = {} item_to_value = functools.partial( _item_to_entry, loggers=loggers) - iterator = HTTPIterator( - client=self._client, path=path, - item_to_value=item_to_value, items_key='entries', - page_token=page_token, extra_params=extra_params) + iterator = page_iterator.HTTPIterator( + client=self._client, + api_request=self._client._connection.api_request, + path=path, + item_to_value=item_to_value, + items_key='entries', + page_token=page_token, + extra_params=extra_params) # This method uses POST to make a read-only request. iterator._HTTP_METHOD = 'POST' return iterator @@ -205,7 +209,7 @@ def list_sinks(self, project, page_size=None, page_token=None): passed, the API will return the first page of sinks. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.logging.sink.Sink` accessible to the current API. @@ -216,10 +220,14 @@ def list_sinks(self, project, page_size=None, page_token=None): extra_params['pageSize'] = page_size path = '/projects/%s/sinks' % (project,) - return HTTPIterator( - client=self._client, path=path, - item_to_value=_item_to_sink, items_key='sinks', - page_token=page_token, extra_params=extra_params) + return page_iterator.HTTPIterator( + client=self._client, + api_request=self._client._connection.api_request, + path=path, + item_to_value=_item_to_sink, + items_key='sinks', + page_token=page_token, + extra_params=extra_params) def sink_create(self, project, sink_name, filter_, destination): """API call: create a sink resource. @@ -345,7 +353,7 @@ def list_metrics(self, project, page_size=None, page_token=None): passed, the API will return the first page of metrics. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.logging.metric.Metric` accessible to the current API. @@ -356,10 +364,14 @@ def list_metrics(self, project, page_size=None, page_token=None): extra_params['pageSize'] = page_size path = '/projects/%s/metrics' % (project,) - return HTTPIterator( - client=self._client, path=path, - item_to_value=_item_to_metric, items_key='metrics', - page_token=page_token, extra_params=extra_params) + return page_iterator.HTTPIterator( + client=self._client, + api_request=self._client._connection.api_request, + path=path, + item_to_value=_item_to_metric, + items_key='metrics', + page_token=page_token, + extra_params=extra_params) def metric_create(self, project, metric_name, filter_, description=None): """API call: create a metric resource. @@ -459,12 +471,12 @@ def _item_to_entry(iterator, resource, loggers): This method does not have the correct signature to be used as the ``item_to_value`` argument to - :class:`~google.cloud.iterator.Iterator`. It is intended to be + :class:`~google.api.core.page_iterator.Iterator`. It is intended to be patched with a mutable ``loggers`` argument that can be updated on subsequent calls. For an example, see how the method is used above in :meth:`_LoggingAPI.list_entries`. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict @@ -485,7 +497,7 @@ def _item_to_entry(iterator, resource, loggers): def _item_to_sink(iterator, resource): """Convert a sink resource to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict @@ -500,7 +512,7 @@ def _item_to_sink(iterator, resource): def _item_to_metric(iterator, resource): """Convert a metric resource to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict diff --git a/logging/google/cloud/logging/client.py b/logging/google/cloud/logging/client.py index 3ce67fba151c..23ec84ec67d0 100644 --- a/logging/google/cloud/logging/client.py +++ b/logging/google/cloud/logging/client.py @@ -194,7 +194,7 @@ def list_entries(self, projects=None, filter_=None, order_by=None, passed, the API will return the first page of entries. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.logging.entries._BaseEntry` accessible to the current client. """ @@ -243,7 +243,7 @@ def list_sinks(self, page_size=None, page_token=None): passed, the API will return the first page of sinks. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.logging.sink.Sink` accessible to the current client. @@ -288,7 +288,7 @@ def list_metrics(self, page_size=None, page_token=None): passed, the API will return the first page of metrics. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.logging.metric.Metric` accessible to the current client. """ diff --git a/logging/google/cloud/logging/logger.py b/logging/google/cloud/logging/logger.py index a13b06cd260b..1006ebb1e693 100644 --- a/logging/google/cloud/logging/logger.py +++ b/logging/google/cloud/logging/logger.py @@ -344,7 +344,7 @@ def list_entries(self, projects=None, filter_=None, order_by=None, passed, the API will return the first page of entries. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.logging.entries._BaseEntry` accessible to the current logger. """ diff --git a/pubsub/google/cloud/pubsub/_gax.py b/pubsub/google/cloud/pubsub/_gax.py index 94dc639178ef..35e56717b3c2 100644 --- a/pubsub/google/cloud/pubsub/_gax.py +++ b/pubsub/google/cloud/pubsub/_gax.py @@ -16,6 +16,7 @@ import functools +from google.api.core import page_iterator from google.cloud.gapic.pubsub.v1.publisher_client import PublisherClient from google.cloud.gapic.pubsub.v1.subscriber_client import SubscriberClient from google.gax import CallOptions @@ -35,7 +36,6 @@ from google.cloud._http import DEFAULT_USER_AGENT from google.cloud.exceptions import Conflict from google.cloud.exceptions import NotFound -from google.cloud.iterator import GAXIterator from google.cloud.pubsub import __version__ from google.cloud.pubsub._helpers import subscription_name_from_path from google.cloud.pubsub.snapshot import Snapshot @@ -78,7 +78,7 @@ def list_topics(self, project, page_size=0, page_token=None): passed, the API will return the first page of topics. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.topic.Topic` accessible to the current API. """ @@ -88,7 +88,8 @@ def list_topics(self, project, page_size=0, page_token=None): path = 'projects/%s' % (project,) page_iter = self._gax_api.list_topics( path, page_size=page_size, options=options) - return GAXIterator(self._client, page_iter, _item_to_topic) + return page_iterator._GAXIterator( + self._client, page_iter, _item_to_topic) def topic_create(self, topic_path): """API call: create a topic @@ -204,7 +205,7 @@ def topic_list_subscriptions(self, topic, page_size=0, page_token=None): If not passed, the API will return the first page of subscriptions. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.subscription.Subscription` accessible to the current API. @@ -223,8 +224,8 @@ def topic_list_subscriptions(self, topic, page_size=0, page_token=None): raise NotFound(topic_path) raise - iterator = GAXIterator(self._client, page_iter, - _item_to_subscription_for_topic) + iterator = page_iterator._GAXIterator( + self._client, page_iter, _item_to_subscription_for_topic) iterator.topic = topic return iterator @@ -260,7 +261,7 @@ def list_subscriptions(self, project, page_size=0, page_token=None): If not passed, the API will return the first page of subscriptions. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.subscription.Subscription` accessible to the current API. @@ -278,7 +279,8 @@ def list_subscriptions(self, project, page_size=0, page_token=None): topics = {} item_to_value = functools.partial( _item_to_sub_for_client, topics=topics) - return GAXIterator(self._client, page_iter, item_to_value) + return page_iterator._GAXIterator( + self._client, page_iter, item_to_value) def subscription_create(self, subscription_path, topic_path, ack_deadline=None, push_endpoint=None, @@ -542,7 +544,7 @@ def list_snapshots(self, project, page_size=0, page_token=None): passed, the API will return the first page of topics. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.snapshot.Snapshot` accessible to the current API. """ @@ -559,7 +561,8 @@ def list_snapshots(self, project, page_size=0, page_token=None): topics = {} item_to_value = functools.partial( _item_to_snapshot_for_client, topics=topics) - return GAXIterator(self._client, page_iter, item_to_value) + return page_iterator._GAXIterator( + self._client, page_iter, item_to_value) def snapshot_create(self, snapshot_path, subscription_path): """API call: create a snapshot @@ -709,7 +712,7 @@ def make_gax_subscriber_api(credentials=None, host=None): def _item_to_topic(iterator, resource): """Convert a protobuf topic to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: :class:`.pubsub_pb2.Topic` @@ -725,7 +728,7 @@ def _item_to_topic(iterator, resource): def _item_to_subscription_for_topic(iterator, subscription_path): """Convert a subscription name to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type subscription_path: str @@ -746,12 +749,12 @@ def _item_to_sub_for_client(iterator, sub_pb, topics): This method does not have the correct signature to be used as the ``item_to_value`` argument to - :class:`~google.cloud.iterator.Iterator`. It is intended to be + :class:`~google.api.core.page_iterator.Iterator`. It is intended to be patched with a mutable topics argument that can be updated on subsequent calls. For an example, see how the method is used above in :meth:`_SubscriberAPI.list_subscriptions`. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type sub_pb: :class:`.pubsub_pb2.Subscription` @@ -776,12 +779,12 @@ def _item_to_snapshot_for_client(iterator, snapshot_pb, topics): This method does not have the correct signature to be used as the ``item_to_value`` argument to - :class:`~google.cloud.iterator.Iterator`. It is intended to be + :class:`~google.api.core.page_iterator.Iterator`. It is intended to be patched with a mutable topics argument that can be updated on subsequent calls. For an example, see how the method is used above in :meth:`_SubscriberAPI.list_snapshots`. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type sub_pb: :class:`.pubsub_pb2.Snapshot` diff --git a/pubsub/google/cloud/pubsub/_http.py b/pubsub/google/cloud/pubsub/_http.py index f1d07237d7df..5173b4095ca8 100644 --- a/pubsub/google/cloud/pubsub/_http.py +++ b/pubsub/google/cloud/pubsub/_http.py @@ -19,10 +19,10 @@ import functools import os +from google.api.core import page_iterator from google.cloud import _http from google.cloud._helpers import _timedelta_to_duration_pb from google.cloud.environment_vars import PUBSUB_EMULATOR -from google.cloud.iterator import HTTPIterator from google.cloud.pubsub import __version__ from google.cloud.pubsub._helpers import subscription_name_from_path @@ -131,7 +131,7 @@ def list_topics(self, project, page_size=None, page_token=None): passed, the API will return the first page of topics. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.topic.Topic` accessible to the current client. """ @@ -140,9 +140,13 @@ def list_topics(self, project, page_size=None, page_token=None): extra_params['pageSize'] = page_size path = '/projects/%s/topics' % (project,) - return HTTPIterator( - client=self._client, path=path, item_to_value=_item_to_topic, - items_key='topics', page_token=page_token, + return page_iterator.HTTPIterator( + client=self._client, + api_request=self._client._connection.api_request, + path=path, + item_to_value=_item_to_topic, + items_key='topics', + page_token=page_token, extra_params=extra_params) def topic_create(self, topic_path): @@ -237,11 +241,14 @@ def topic_list_subscriptions(self, topic, page_size=None, page_token=None): extra_params['pageSize'] = page_size path = '/%s/subscriptions' % (topic.full_name,) - iterator = HTTPIterator( - client=self._client, path=path, + iterator = page_iterator.HTTPIterator( + client=self._client, + api_request=self._client._connection.api_request, + path=path, item_to_value=_item_to_subscription_for_topic, items_key='subscriptions', - page_token=page_token, extra_params=extra_params) + page_token=page_token, + extra_params=extra_params) iterator.topic = topic return iterator @@ -275,7 +282,7 @@ def list_subscriptions(self, project, page_size=None, page_token=None): If not passed, the API will return the first page of subscriptions. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.subscription.Subscription` accessible to the current API. @@ -291,9 +298,13 @@ def list_subscriptions(self, project, page_size=None, page_token=None): topics = {} item_to_value = functools.partial( _item_to_sub_for_client, topics=topics) - return HTTPIterator( - client=self._client, path=path, item_to_value=item_to_value, - items_key='subscriptions', page_token=page_token, + return page_iterator.HTTPIterator( + client=self._client, + api_request=self._client._connection.api_request, + path=path, + item_to_value=item_to_value, + items_key='subscriptions', + page_token=page_token, extra_params=extra_params) def subscription_create(self, subscription_path, topic_path, @@ -536,7 +547,7 @@ def list_snapshots(self, project, page_size=None, page_token=None): passed, the API will return the first page of topics. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.snapshot.Snapshot` accessible to the current API. """ @@ -551,9 +562,13 @@ def list_snapshots(self, project, page_size=None, page_token=None): topics = {} item_to_value = functools.partial( _item_to_snapshot_for_client, topics=topics) - return HTTPIterator( - client=self._client, path=path, item_to_value=item_to_value, - items_key='snapshots', page_token=page_token, + return page_iterator.HTTPIterator( + client=self._client, + api_request=self._client._connection.api_request, + path=path, + item_to_value=item_to_value, + items_key='snapshots', + page_token=page_token, extra_params=extra_params) def snapshot_create(self, snapshot_path, subscription_path): @@ -695,7 +710,7 @@ def _transform_messages_base64(messages, transform, key=None): def _item_to_topic(iterator, resource): """Convert a JSON topic to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict @@ -710,7 +725,7 @@ def _item_to_topic(iterator, resource): def _item_to_subscription_for_topic(iterator, subscription_path): """Convert a subscription name to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type subscription_path: str @@ -731,12 +746,12 @@ def _item_to_sub_for_client(iterator, resource, topics): This method does not have the correct signature to be used as the ``item_to_value`` argument to - :class:`~google.cloud.iterator.Iterator`. It is intended to be + :class:`~google.api.core.page_iterator.Iterator`. It is intended to be patched with a mutable topics argument that can be updated on subsequent calls. For an example, see how the method is used above in :meth:`_SubscriberAPI.list_subscriptions`. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict @@ -760,12 +775,12 @@ def _item_to_snapshot_for_client(iterator, resource, topics): This method does not have the correct signature to be used as the ``item_to_value`` argument to - :class:`~google.cloud.iterator.Iterator`. It is intended to be + :class:`~google.api.core.page_iterator.Iterator`. It is intended to be patched with a mutable topics argument that can be updated on subsequent calls. For an example, see how the method is used above in :meth:`_SubscriberAPI.list_snapshots`. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type resource: dict diff --git a/pubsub/google/cloud/pubsub/client.py b/pubsub/google/cloud/pubsub/client.py index ae808c038b7c..0dc9b8fb6f38 100644 --- a/pubsub/google/cloud/pubsub/client.py +++ b/pubsub/google/cloud/pubsub/client.py @@ -154,7 +154,7 @@ def list_topics(self, page_size=None, page_token=None): passed, the API will return the first page of topics. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.topic.Topic` accessible to the current API. """ @@ -183,7 +183,7 @@ def list_subscriptions(self, page_size=None, page_token=None): passed, the API will return the first page of topics. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.subscription.Subscription` accessible to the current client. @@ -210,7 +210,7 @@ def list_snapshots(self, page_size=None, page_token=None): passed, the API will return the first page of topics. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.snapshot.Snapshot` accessible to the current API. """ diff --git a/pubsub/google/cloud/pubsub/topic.py b/pubsub/google/cloud/pubsub/topic.py index 92c323ed63d7..92f453bd2b2b 100644 --- a/pubsub/google/cloud/pubsub/topic.py +++ b/pubsub/google/cloud/pubsub/topic.py @@ -330,7 +330,7 @@ def list_subscriptions(self, page_size=None, page_token=None, client=None): :param client: the client to use. If not passed, falls back to the ``client`` stored on the current topic. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.pubsub.subscription.Subscription` accessible to the current topic. diff --git a/resource_manager/google/cloud/resource_manager/client.py b/resource_manager/google/cloud/resource_manager/client.py index 90f187735974..2a4ff91daf21 100644 --- a/resource_manager/google/cloud/resource_manager/client.py +++ b/resource_manager/google/cloud/resource_manager/client.py @@ -15,8 +15,8 @@ """A Client for interacting with the Resource Manager API.""" +from google.api.core import page_iterator from google.cloud.client import Client as BaseClient -from google.cloud.iterator import HTTPIterator from google.cloud.resource_manager._http import Connection from google.cloud.resource_manager.project import Project @@ -151,7 +151,7 @@ def list_projects(self, filter_params=None, page_size=None): single page. If not passed, defaults to a value set by the API. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of all :class:`~google.cloud.resource_manager.project.Project`. that the current user has access to. @@ -164,15 +164,19 @@ def list_projects(self, filter_params=None, page_size=None): if filter_params is not None: extra_params['filter'] = filter_params - return HTTPIterator( - client=self, path='/projects', item_to_value=_item_to_project, - items_key='projects', extra_params=extra_params) + return page_iterator.HTTPIterator( + client=self, + api_request=self._connection.api_request, + path='/projects', + item_to_value=_item_to_project, + items_key='projects', + extra_params=extra_params) def _item_to_project(iterator, resource): """Convert a JSON project to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that has retrieved the item. :type resource: dict diff --git a/resource_manager/tests/unit/test_client.py b/resource_manager/tests/unit/test_client.py index 1ba312775f17..06ac02cbe627 100644 --- a/resource_manager/tests/unit/test_client.py +++ b/resource_manager/tests/unit/test_client.py @@ -88,7 +88,7 @@ def test_fetch_project(self): self.assertEqual(project.labels, labels) def test_list_projects_return_type(self): - from google.cloud.iterator import HTTPIterator + from google.api.core import page_iterator credentials = _make_credentials() client = self._make_one(credentials=credentials) @@ -96,7 +96,7 @@ def test_list_projects_return_type(self): client._connection = _Connection({}) results = client.list_projects() - self.assertIsInstance(results, HTTPIterator) + self.assertIsInstance(results, page_iterator.HTTPIterator) def test_list_projects_no_paging(self): credentials = _make_credentials() @@ -227,12 +227,12 @@ def test_list_projects_with_filter(self): }) def test_page_empty_response(self): - from google.cloud.iterator import Page + from google.api.core import page_iterator credentials = _make_credentials() client = self._make_one(credentials=credentials) iterator = client.list_projects() - page = Page(iterator, (), None) + page = page_iterator.Page(iterator, (), None) iterator._page = page self.assertEqual(page.num_items, 0) self.assertEqual(page.remaining, 0) diff --git a/runtimeconfig/google/cloud/runtimeconfig/config.py b/runtimeconfig/google/cloud/runtimeconfig/config.py index 385b92a31c40..1b86e7971aab 100644 --- a/runtimeconfig/google/cloud/runtimeconfig/config.py +++ b/runtimeconfig/google/cloud/runtimeconfig/config.py @@ -14,10 +14,10 @@ """Create / interact with Google Cloud RuntimeConfig configs.""" +from google.api.core import page_iterator from google.cloud.exceptions import NotFound from google.cloud.runtimeconfig._helpers import config_name_from_full_name from google.cloud.runtimeconfig.variable import Variable -from google.cloud.iterator import HTTPIterator class Config(object): @@ -232,16 +232,21 @@ def list_variables(self, page_size=None, page_token=None, client=None): (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current config. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.runtimeconfig.variable.Variable` belonging to this project. """ path = '%s/variables' % (self.path,) - iterator = HTTPIterator( - client=self._require_client(client), path=path, - item_to_value=_item_to_variable, items_key='variables', - page_token=page_token, max_results=page_size) + client = self._require_client(client) + iterator = page_iterator.HTTPIterator( + client=client, + api_request=client._connection.api_request, + path=path, + item_to_value=_item_to_variable, + items_key='variables', + page_token=page_token, + max_results=page_size) iterator._MAX_RESULTS = 'pageSize' iterator.config = self return iterator @@ -250,7 +255,7 @@ def list_variables(self, page_size=None, page_token=None, client=None): def _item_to_variable(iterator, resource): """Convert a JSON variable to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that has retrieved the item. :type resource: dict diff --git a/spanner/google/cloud/spanner/client.py b/spanner/google/cloud/spanner/client.py index 6274d28d9e18..34e0a81c4fc4 100644 --- a/spanner/google/cloud/spanner/client.py +++ b/spanner/google/cloud/spanner/client.py @@ -24,6 +24,7 @@ :class:`~google.cloud.spanner.database.Database` """ +from google.api.core import page_iterator from google.gax import INITIAL_PAGE # pylint: disable=line-too-long from google.cloud.gapic.spanner_admin_database.v1.database_admin_client import ( # noqa @@ -34,7 +35,6 @@ from google.cloud._http import DEFAULT_USER_AGENT from google.cloud.client import ClientWithProject -from google.cloud.iterator import GAXIterator from google.cloud.spanner import __version__ from google.cloud.spanner._helpers import _options_with_prefix from google.cloud.spanner.instance import DEFAULT_NODE_COUNT @@ -194,7 +194,7 @@ def list_instance_configs(self, page_size=None, page_token=None): :type page_token: str :param page_token: (Optional) Token for fetching next page of results. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.spanner.instance.InstanceConfig` @@ -207,7 +207,8 @@ def list_instance_configs(self, page_size=None, page_token=None): path = 'projects/%s' % (self.project,) page_iter = self.instance_admin_api.list_instance_configs( path, page_size=page_size, options=options) - return GAXIterator(self, page_iter, _item_to_instance_config) + return page_iterator._GAXIterator( + self, page_iter, _item_to_instance_config) def instance(self, instance_id, configuration_name=None, @@ -257,7 +258,7 @@ def list_instances(self, filter_='', page_size=None, page_token=None): :type page_token: str :param page_token: (Optional) Token for fetching next page of results. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.spanner.instance.Instance` resources within the client's project. @@ -269,14 +270,15 @@ def list_instances(self, filter_='', page_size=None, page_token=None): path = 'projects/%s' % (self.project,) page_iter = self.instance_admin_api.list_instances( path, filter_=filter_, page_size=page_size, options=options) - return GAXIterator(self, page_iter, _item_to_instance) + return page_iterator._GAXIterator( + self, page_iter, _item_to_instance) def _item_to_instance_config( iterator, config_pb): # pylint: disable=unused-argument """Convert an instance config protobuf to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type config_pb: @@ -292,7 +294,7 @@ def _item_to_instance_config( def _item_to_instance(iterator, instance_pb): """Convert an instance protobuf to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type instance_pb: :class:`~google.spanner.admin.instance.v1.Instance` diff --git a/spanner/google/cloud/spanner/instance.py b/spanner/google/cloud/spanner/instance.py index 7d715c94c590..34cb5b1b0bc2 100644 --- a/spanner/google/cloud/spanner/instance.py +++ b/spanner/google/cloud/spanner/instance.py @@ -16,6 +16,7 @@ import re +from google.api.core import page_iterator from google.gax import INITIAL_PAGE from google.gax.errors import GaxError from google.gax.grpc import exc_to_code @@ -27,7 +28,6 @@ # pylint: disable=ungrouped-imports from google.cloud.exceptions import Conflict from google.cloud.exceptions import NotFound -from google.cloud.iterator import GAXIterator from google.cloud.spanner._helpers import _options_with_prefix from google.cloud.spanner.database import Database from google.cloud.spanner.pool import BurstyPool @@ -374,7 +374,7 @@ def list_databases(self, page_size=None, page_token=None): :type page_token: str :param page_token: (Optional) Token for fetching next page of results. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of :class:`~google.cloud.spanner.database.Database` resources within the current instance. @@ -384,7 +384,8 @@ def list_databases(self, page_size=None, page_token=None): options = _options_with_prefix(self.name, page_token=page_token) page_iter = self._client.database_admin_api.list_databases( self.name, page_size=page_size, options=options) - iterator = GAXIterator(self._client, page_iter, _item_to_database) + iterator = page_iterator._GAXIterator( + self._client, page_iter, _item_to_database) iterator.instance = self return iterator @@ -392,7 +393,7 @@ def list_databases(self, page_size=None, page_token=None): def _item_to_database(iterator, database_pb): """Convert a database protobuf to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type database_pb: :class:`~google.spanner.admin.database.v1.Database` diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index 06550b09ffcb..ad3a20e3aef0 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -22,12 +22,12 @@ import google.auth.credentials import six +from google.api.core import page_iterator from google.cloud._helpers import _datetime_to_rfc3339 from google.cloud._helpers import _NOW from google.cloud._helpers import _rfc3339_to_datetime from google.cloud.exceptions import NotFound from google.cloud.iam import Policy -from google.cloud.iterator import HTTPIterator from google.cloud.storage._helpers import _PropertyMixin from google.cloud.storage._helpers import _scalar_property from google.cloud.storage._helpers import _validate_name @@ -40,7 +40,7 @@ def _blobs_page_start(iterator, page, response): """Grab prefixes after a :class:`~google.cloud.iterator.Page` started. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type page: :class:`~google.cloud.iterator.Page` @@ -61,7 +61,7 @@ def _item_to_blob(iterator, item): This assumes that the ``bucket`` attribute has been added to the iterator after being created. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that has retrieved the item. :type item: dict @@ -316,7 +316,7 @@ def list_blobs(self, max_results=None, page_token=None, prefix=None, :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of all :class:`~google.cloud.storage.blob.Blob` in this bucket matching the arguments. """ @@ -338,10 +338,15 @@ def list_blobs(self, max_results=None, page_token=None, prefix=None, client = self._require_client(client) path = self.path + '/o' - iterator = HTTPIterator( - client=client, path=path, item_to_value=_item_to_blob, - page_token=page_token, max_results=max_results, - extra_params=extra_params, page_start=_blobs_page_start) + iterator = page_iterator.HTTPIterator( + client=client, + api_request=client._connection.api_request, + path=path, + item_to_value=_item_to_blob, + page_token=page_token, + max_results=max_results, + extra_params=extra_params, + page_start=_blobs_page_start) iterator.bucket = self iterator.prefixes = set() return iterator diff --git a/storage/google/cloud/storage/client.py b/storage/google/cloud/storage/client.py index 42b4bb7d9592..5743dc059936 100644 --- a/storage/google/cloud/storage/client.py +++ b/storage/google/cloud/storage/client.py @@ -15,10 +15,10 @@ """Client for interacting with the Google Cloud Storage API.""" +from google.api.core import page_iterator from google.cloud._helpers import _LocalStack from google.cloud.client import ClientWithProject from google.cloud.exceptions import NotFound -from google.cloud.iterator import HTTPIterator from google.cloud.storage._http import Connection from google.cloud.storage.batch import Batch from google.cloud.storage.bucket import Bucket @@ -255,7 +255,7 @@ def list_buckets(self, max_results=None, page_token=None, prefix=None, response with just the next page token and the language of each bucket returned: 'items/id,nextPageToken' - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket` belonging to this project. """ @@ -269,16 +269,20 @@ def list_buckets(self, max_results=None, page_token=None, prefix=None, if fields is not None: extra_params['fields'] = fields - return HTTPIterator( - client=self, path='/b', item_to_value=_item_to_bucket, - page_token=page_token, max_results=max_results, + return page_iterator.HTTPIterator( + client=self, + api_request=self._connection.api_request, + path='/b', + item_to_value=_item_to_bucket, + page_token=page_token, + max_results=max_results, extra_params=extra_params) def _item_to_bucket(iterator, item): """Convert a JSON bucket to the native object. - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that has retrieved the item. :type item: dict diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py index 0df94dc5db3d..ece5c5411773 100644 --- a/storage/tests/unit/test_bucket.py +++ b/storage/tests/unit/test_bucket.py @@ -1190,14 +1190,14 @@ def test_make_public_recursive_too_many(self): self.assertRaises(ValueError, bucket.make_public, recursive=True) def test_page_empty_response(self): - from google.cloud.iterator import Page + from google.api.core import page_iterator connection = _Connection() client = _Client(connection) name = 'name' bucket = self._make_one(client=client, name=name) iterator = bucket.list_blobs() - page = Page(iterator, (), None) + page = page_iterator.Page(iterator, (), None) iterator._page = page blobs = list(page) self.assertEqual(blobs, []) diff --git a/storage/tests/unit/test_client.py b/storage/tests/unit/test_client.py index ab75d9be8fca..39a27b9c0773 100644 --- a/storage/tests/unit/test_client.py +++ b/storage/tests/unit/test_client.py @@ -404,13 +404,13 @@ def test_list_buckets_all_arguments(self): self.assertEqual(parse_qs(uri_parts.query), expected_query) def test_page_empty_response(self): - from google.cloud.iterator import Page + from google.api.core import page_iterator project = 'PROJECT' credentials = _make_credentials() client = self._make_one(project=project, credentials=credentials) iterator = client.list_buckets() - page = Page(iterator, (), None) + page = page_iterator.Page(iterator, (), None) iterator._page = page self.assertEqual(list(page), []) diff --git a/trace/google/cloud/trace/_gax.py b/trace/google/cloud/trace/_gax.py index b412b54ee856..8ba4a456768f 100644 --- a/trace/google/cloud/trace/_gax.py +++ b/trace/google/cloud/trace/_gax.py @@ -14,13 +14,13 @@ """GAX Wrapper for interacting with the Stackdriver Trace API.""" +from google.api.core import page_iterator from google.cloud.gapic.trace.v1 import trace_service_client from google.cloud.proto.devtools.cloudtrace.v1 import trace_pb2 from google.gax import CallOptions from google.gax import INITIAL_PAGE from google.cloud._helpers import make_secure_channel from google.cloud._http import DEFAULT_USER_AGENT -from google.cloud.iterator import GAXIterator from google.protobuf.json_format import MessageToDict from google.protobuf.json_format import ParseDict @@ -131,7 +131,7 @@ def list_traces( passed, the API will return the first page of entries. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Traces that match the specified filter conditions. """ if page_token is None: @@ -147,7 +147,8 @@ def list_traces( order_by=order_by, options=options) item_to_value = _item_to_mapping - return GAXIterator(self.client, page_iter, item_to_value) + return page_iterator._GAXIterator( + self.client, page_iter, item_to_value) def _parse_trace_pb(trace_pb): @@ -169,7 +170,7 @@ def _parse_trace_pb(trace_pb): def _item_to_mapping(iterator, trace_pb): """Helper callable function for the GAXIterator - :type iterator: :class:`~google.cloud.iterator.Iterator` + :type iterator: :class:`~google.api.core.page_iterator.Iterator` :param iterator: The iterator that is currently in use. :type trace_pb: :class:`google.cloud.proto.devtools.cloudtrace.v1. diff --git a/trace/google/cloud/trace/client.py b/trace/google/cloud/trace/client.py index d2104924a2bf..96635e14e107 100644 --- a/trace/google/cloud/trace/client.py +++ b/trace/google/cloud/trace/client.py @@ -144,7 +144,7 @@ def list_traces( passed, the API will return the first page of entries. - :rtype: :class:`~google.cloud.iterator.Iterator` + :rtype: :class:`~google.api.core.page_iterator.Iterator` :returns: Traces that match the specified filter conditions. """ if project_id is None: