Fix system test under Datastore Emulator. (Fixes #118) (#119)

Improve usage of Datastore Emulator by not requiring credentials to be
set.

Rewrite failing system test to work around emulator discrepency with
``more_results`` field of ``QueryResultsBatch`` message.

See: googleapis/google-cloud-datastore#130

Update the ``more`` return value of ``Query.fetch_page`` to be ``False``
if an empty page has just been retrieved. This is intended to prevent
possible infinite loops in client code when using the Datastore
emulator.

src/google/cloud/ndb/client.py

@@ -16,6 +16,7 @@
 
 import contextlib
 import os
+import requests
 
 from google.cloud import environment_vars
 from google.cloud import _helpers
@@ -80,7 +81,6 @@ class Client(google_client.ClientWithProject):
     """The scopes required for authenticating as a Cloud Datastore consumer."""
 
     def __init__(self, project=None, namespace=None, credentials=None):
-        super(Client, self).__init__(project=project, credentials=credentials)
         self.namespace = namespace
         self.host = os.environ.get(
             environment_vars.GCD_HOST, DATASTORE_API_HOST
@@ -91,6 +91,22 @@ def __init__(self, project=None, namespace=None, credentials=None):
         emulator = bool(os.environ.get("DATASTORE_EMULATOR_HOST"))
         self.secure = not emulator
 
+        if emulator:
+            # When using the emulator, in theory, the client shouldn't need to
+            # call home to authenticate, as you don't need to authenticate to
+            # use the local emulator. Unfortunately, the client calls home to
+            # authenticate anyway, unless you pass ``requests.Session`` to
+            # ``_http`` which seems to be the preferred work around.
+            super(Client, self).__init__(
+                project=project,
+                credentials=credentials,
+                _http=requests.Session,
+            )
+        else:
+            super(Client, self).__init__(
+                project=project, credentials=credentials
+            )
+
     @contextlib.contextmanager
     def context(self, cache_policy=None):
         """Establish a context for a set of NDB calls.

src/google/cloud/ndb/query.py

@@ -2279,7 +2279,7 @@ def fetch_page_async(
             results.append(result.entity())
             cursor = result.cursor
 
-        more = (
+        more = results and (
             iterator._more_results_after_limit or iterator.probably_has_next()
         )
         return results, cursor, more

tests/system/conftest.py

@@ -1,14 +1,26 @@
 import itertools
+import os
 import uuid
 
 import pytest
+import requests
 
 from google.cloud import datastore
 from google.cloud import ndb
 
 from . import KIND, OTHER_KIND, OTHER_NAMESPACE
 
 
+def _make_ds_client(namespace):
+    emulator = bool(os.environ.get("DATASTORE_EMULATOR_HOST"))
+    if emulator:
+        client = datastore.Client(namespace=namespace, _http=requests.Session)
+    else:
+        client = datastore.Client(namespace=namespace)
+
+    return client
+
+
 def all_entities(client):
     return itertools.chain(
         client.query(kind=KIND).fetch(),
@@ -20,7 +32,7 @@ def all_entities(client):
 @pytest.fixture(scope="module", autouse=True)
 def initial_clean():
     # Make sure database is in clean state at beginning of test run
-    client = datastore.Client()
+    client = _make_ds_client(None)
     for entity in all_entities(client):
         client.delete(entity.key)
 
@@ -36,39 +48,42 @@ def to_delete():
 
 
 @pytest.fixture
-def ds_client(namespace, to_delete, deleted_keys):
-    client = datastore.Client(namespace=namespace)
+def ds_client(namespace):
+    return _make_ds_client(namespace)
 
+
+@pytest.fixture
+def with_ds_client(ds_client, to_delete, deleted_keys):
     # Make sure we're leaving database as clean as we found it after each test
     results = [
         entity
-        for entity in all_entities(client)
+        for entity in all_entities(ds_client)
         if entity.key not in deleted_keys
     ]
     assert not results
 
-    yield client
+    yield ds_client
 
     if to_delete:
-        client.delete_multi(to_delete)
+        ds_client.delete_multi(to_delete)
         deleted_keys.update(to_delete)
 
     not_deleted = [
         entity
-        for entity in all_entities(client)
+        for entity in all_entities(ds_client)
         if entity.key not in deleted_keys
     ]
     assert not not_deleted
 
 
 @pytest.fixture
-def ds_entity(ds_client, dispose_of):
+def ds_entity(with_ds_client, dispose_of):
     def make_entity(*key_args, **entity_kwargs):
-        key = ds_client.key(*key_args)
-        assert ds_client.get(key) is None
+        key = with_ds_client.key(*key_args)
+        assert with_ds_client.get(key) is None
         entity = datastore.Entity(key=key)
         entity.update(entity_kwargs)
-        ds_client.put(entity)
+        with_ds_client.put(entity)
         dispose_of(key)
 
         return entity
@@ -77,7 +92,7 @@ def make_entity(*key_args, **entity_kwargs):
 
 
 @pytest.fixture
-def dispose_of(ds_client, to_delete):
+def dispose_of(with_ds_client, to_delete):
     def delete_entity(ds_key):
         to_delete.append(ds_key)
 

tests/system/test_crud.py

@@ -22,7 +22,6 @@
 
 import test_utils.system
 
-from google.cloud import datastore
 from google.cloud import ndb
 
 from tests.system import KIND, eventually
@@ -132,7 +131,7 @@ def get_two_entities():
 
 
 @pytest.mark.usefixtures("client_context")
-def test_insert_entity(dispose_of):
+def test_insert_entity(dispose_of, ds_client):
     class SomeKind(ndb.Model):
         foo = ndb.IntegerProperty()
         bar = ndb.StringProperty()
@@ -145,7 +144,6 @@ class SomeKind(ndb.Model):
     assert retrieved.bar == "none"
 
     # Make sure strings are stored as strings in datastore
-    ds_client = datastore.Client()
     ds_entity = ds_client.get(key._key)
     assert ds_entity["bar"] == "none"
 

tests/system/test_query.py

@@ -522,6 +522,9 @@ def make_entities():
         page_size, start_cursor=next_cursor
     )
     assert [entity.foo for entity in results] == [5, 6, 7, 8, 9]
+
+    results, cursor, more = query.fetch_page(page_size, start_cursor=cursor)
+    assert not results
     assert not more
 
 

tests/unit/test_query.py

@@ -1958,6 +1958,33 @@ def next(self):
             raw=True,
         )
 
+    @staticmethod
+    @pytest.mark.usefixtures("in_context")
+    @unittest.mock.patch("google.cloud.ndb.query._datastore_query")
+    def test_fetch_page_beyond_last_page(_datastore_query):
+        class DummyQueryIterator:
+            # Emulates the Datastore emulator behavior
+            _more_results_after_limit = True
+
+            def __init__(self):
+                self.items = []
+
+            def has_next_async(self):
+                return utils.future_result(False)
+
+        _datastore_query.iterate.return_value = DummyQueryIterator()
+        query = query_module.Query()
+        results, cursor, more = query.fetch_page(5, start_cursor="cursor000")
+        assert results == []
+        assert not more
+
+        _datastore_query.iterate.assert_called_once_with(
+            query_module.QueryOptions(
+                project="testing", limit=5, start_cursor="cursor000"
+            ),
+            raw=True,
+        )
+
     @staticmethod
     @pytest.mark.usefixtures("in_context")
     @unittest.mock.patch("google.cloud.ndb.query._datastore_query")