diff --git a/docs/global_cache.rst b/docs/global_cache.rst new file mode 100644 index 00000000..80c384d6 --- /dev/null +++ b/docs/global_cache.rst @@ -0,0 +1,7 @@ +####### +Context +####### + +.. automodule:: google.cloud.ndb.global_cache + :members: + :show-inheritance: diff --git a/docs/index.rst b/docs/index.rst index c98c661c..10e4a4de 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,6 +8,7 @@ client context + global_cache key model query diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index e1f311b5..e5c5d1e6 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -62,6 +62,7 @@ prefetch protobuf proxied QueryOptions +Redis RequestHandler runtime schemas diff --git a/src/google/cloud/ndb/__init__.py b/src/google/cloud/ndb/__init__.py index 04390df5..4c839eb8 100644 --- a/src/google/cloud/ndb/__init__.py +++ b/src/google/cloud/ndb/__init__.py @@ -52,6 +52,7 @@ "get_indexes_async", "get_multi", "get_multi_async", + "GlobalCache", "in_transaction", "Index", "IndexProperty", @@ -135,6 +136,7 @@ from google.cloud.ndb._datastore_api import STRONG from google.cloud.ndb._datastore_query import Cursor from google.cloud.ndb._datastore_query import QueryIterator +from google.cloud.ndb.global_cache import GlobalCache from google.cloud.ndb.key import Key from google.cloud.ndb.model import BlobKey from google.cloud.ndb.model import BlobKeyProperty diff --git a/src/google/cloud/ndb/_cache.py b/src/google/cloud/ndb/_cache.py index 74f5fd00..10e42c1b 100644 --- a/src/google/cloud/ndb/_cache.py +++ b/src/google/cloud/ndb/_cache.py @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import abc import collections import itertools -import time from google.cloud.ndb import _batch from google.cloud.ndb import context as context_module @@ -46,73 +44,6 @@ def get_and_validate(self, key): raise KeyError(key) -class GlobalCache(abc.ABC): - """Abstract base class for a global entity cache. - - A global entity cache is shared across contexts, sessions, and possibly - even servers. A concrete implementation is available which uses Redis. - """ - - @abc.abstractmethod - def get(self, keys): - """Retrieve entities from the cache. - - Arguments: - keys (List[bytes]): The keys to get. - - Returns: - List[Union[bytes, None]]]: Serialized entities, or :data:`None`, - for each key. - """ - raise NotImplementedError - - @abc.abstractmethod - def set(self, items, expires=None): - """Store entities in the cache. - - Arguments: - items (Dict[bytes, Union[bytes, None]]): Mapping of keys to - serialized entities. - expires (Optional[float]): Number of seconds until value expires. - """ - raise NotImplementedError - - @abc.abstractmethod - def delete(self, keys): - """Remove entities from the cache. - - Arguments: - keys (List[bytes]): The keys to remove. - """ - raise NotImplementedError - - @abc.abstractmethod - def watch(self, keys): - """Begin an optimistic transaction for the given keys. - - A future call to :meth:`compare_and_swap` will only set values for keys - whose values haven't changed since the call to this method. - - Arguments: - keys (List[bytes]): The keys to watch. - """ - raise NotImplementedError - - @abc.abstractmethod - def compare_and_swap(self, items, expires=None): - """Like :meth:`set` but using an optimistic transaction. - - Only keys whose values haven't changed since a preceeding call to - :meth:`watch` will be changed. - - Arguments: - items (Dict[bytes, Union[bytes, None]]): Mapping of keys to - serialized entities. - expires (Optional[float]): Number of seconds until value expires. - """ - raise NotImplementedError - - def _future_result(result): """Returns a completed Future with the given result. @@ -124,89 +55,26 @@ def _future_result(result): return future -class _InProcessGlobalCache(GlobalCache): - """Reference implementation of :class:`GlobalCache`. - - Not intended for production use. Uses a single process wide dictionary to - keep an in memory cache. For use in testing and to have an easily grokkable - reference implementation. Thread safety is potentially a little sketchy. - """ - - cache = {} - """Dict: The cache. - - Relies on atomicity of ``__setitem__`` for thread safety. See: - http://effbot.org/pyfaq/what-kinds-of-global-value-mutation-are-thread-safe.htm +class _GlobalCacheBatch: + """Abstract base for classes used to batch operations for the global cache. """ - def __init__(self): - self._watch_keys = {} - - def get(self, keys): - """Implements :meth:`GlobalCache.get`.""" - now = time.time() - results = [self.cache.get(key) for key in keys] - entity_pbs = [] - for result in results: - if result is not None: - entity_pb, expires = result - if expires and expires < now: - entity_pb = None - else: - entity_pb = None - - entity_pbs.append(entity_pb) - - return _future_result(entity_pbs) - - def set(self, items, expires=None): - """Implements :meth:`GlobalCache.set`.""" - if expires: - expires = time.time() + expires - - for key, value in items.items(): - self.cache[key] = (value, expires) # Supposedly threadsafe - - return _future_result(None) - - def delete(self, keys): - """Implements :meth:`GlobalCache.delete`.""" - for key in keys: - self.cache.pop(key, None) # Threadsafe? - - return _future_result(None) - - def watch(self, keys): - """Implements :meth:`GlobalCache.watch`.""" - for key in keys: - self._watch_keys[key] = self.cache.get(key) - - return _future_result(None) - - def compare_and_swap(self, items, expires=None): - """Implements :meth:`GlobalCache.compare_and_swap`.""" - if expires: - expires = time.time() + expires - - for key, new_value in items.items(): - watch_value = self._watch_keys.get(key) - current_value = self.cache.get(key) - if watch_value == current_value: - self.cache[key] = (new_value, expires) - - return _future_result(None) - - -class _GlobalCacheBatch: def idle_callback(self): - """Get keys from the global cache.""" + """Call the cache operation. + + Also, schedule a callback for the completed operation. + """ cache_call = self.make_call() if not isinstance(cache_call, tasklets.Future): cache_call = _future_result(cache_call) cache_call.add_done_callback(self.done_callback) def done_callback(self, cache_call): - """Process results of call to global cache.""" + """Process results of call to global cache. + + If there is an exception for the cache call, distribute that to waiting + futures, otherwise set the result for all waiting futures to ``None``. + """ exception = cache_call.exception() if exception: for future in self.futures: @@ -216,6 +84,14 @@ def done_callback(self, cache_call): for future in self.futures: future.set_result(None) + def make_call(self): + """Make the actual call to the global cache. To be overridden.""" + raise NotImplementedError + + def future_info(self, key): + """Generate info string for Future. To be overridden.""" + raise NotImplementedError + def global_get(key): """Get entity from global cache. @@ -265,7 +141,12 @@ def add(self, key): return future def done_callback(self, cache_call): - """Process results of call to global cache.""" + """Process results of call to global cache. + + If there is an exception for the cache call, distribute that to waiting + futures, otherwise distribute cache hits or misses to their respective + waiting futures. + """ exception = cache_call.exception() if exception: for future in itertools.chain(*self.todo.values()): @@ -280,12 +161,12 @@ def done_callback(self, cache_call): future.set_result(result) def make_call(self): - """Make the actual call. To be overridden.""" + """Call :method:`GlobalCache.get`.""" cache = context_module.get_context().global_cache return cache.get(self.todo.keys()) def future_info(self, key): - """Generate info string for Future. To be overridden.""" + """Generate info string for Future.""" return "GlobalCache.get({})".format(key) @@ -332,12 +213,12 @@ def add(self, key, value): return future def make_call(self): - """Make the actual call. To be overridden.""" + """Call :method:`GlobalCache.set`.""" cache = context_module.get_context().global_cache return cache.set(self.todo, expires=self.expires) def future_info(self, key, value): - """Generate info string for Future. To be overridden.""" + """Generate info string for Future.""" return "GlobalCache.set({}, {})".format(key, value) @@ -355,7 +236,7 @@ def global_delete(key): class _GlobalCacheDeleteBatch(_GlobalCacheBatch): - """Batch for global cache set requests. """ + """Batch for global cache delete requests.""" def __init__(self, ignore_options): self.keys = [] @@ -376,12 +257,12 @@ def add(self, key): return future def make_call(self): - """Make the actual call. To be overridden.""" + """Call :method:`GlobalCache.delete`.""" cache = context_module.get_context().global_cache return cache.delete(self.keys) def future_info(self, key): - """Generate info string for Future. To be overridden.""" + """Generate info string for Future.""" return "GlobalCache.delete({})".format(key) @@ -409,12 +290,12 @@ def __init__(self, ignore_options): self.futures = [] def make_call(self): - """Make the actual call. To be overridden.""" + """Call :method:`GlobalCache.watch`.""" cache = context_module.get_context().global_cache return cache.watch(self.keys) def future_info(self, key): - """Generate info string for Future. To be overridden.""" + """Generate info string for Future.""" return "GlobalWatch.delete({})".format(key) @@ -444,12 +325,12 @@ class _GlobalCacheCompareAndSwapBatch(_GlobalCacheSetBatch): """Batch for global cache compare and swap requests. """ def make_call(self): - """Make the actual call. To be overridden.""" + """Call :method:`GlobalCache.compare_and_swap`.""" cache = context_module.get_context().global_cache return cache.compare_and_swap(self.todo, expires=self.expires) def future_info(self, key, value): - """Generate info string for Future. To be overridden.""" + """Generate info string for Future.""" return "GlobalCache.compare_and_swap({}, {})".format(key, value) diff --git a/src/google/cloud/ndb/client.py b/src/google/cloud/ndb/client.py index 0a561ca7..5b195a48 100644 --- a/src/google/cloud/ndb/client.py +++ b/src/google/cloud/ndb/client.py @@ -148,9 +148,9 @@ def context( cache_policy (Optional[Callable[[key.Key], bool]]): The cache policy to use in this context. See: :meth:`~google.cloud.ndb.context.Context.set_cache_policy`. - global_cache (Optional[_cache.GlobalCache]): + global_cache (Optional[global_cache.GlobalCache]): The global cache for this context. See: - :class:`~google.cloud.ndb._cache.GlobalCache`. + :class:`~google.cloud.ndb.global_cache.GlobalCache`. global_cache_policy (Optional[Callable[[key.Key], bool]]): The global cache policy to use in this context. See: :meth:`~google.cloud.ndb.context.Context.set_global_cache_policy`. diff --git a/src/google/cloud/ndb/global_cache.py b/src/google/cloud/ndb/global_cache.py new file mode 100644 index 00000000..987b35b8 --- /dev/null +++ b/src/google/cloud/ndb/global_cache.py @@ -0,0 +1,162 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import time + +"""GlobalCache interface and its implementations.""" + + +class GlobalCache(abc.ABC): + """Abstract base class for a global entity cache. + + A global entity cache is shared across contexts, sessions, and possibly + even servers. A concrete implementation is available which uses Redis. + + Essentially, this class models a simple key/value store where keys and + values are arbitrary ``bytes`` instances. "Compare and swap", aka + "optimistic transactions" should also be supported. + + Concrete implementations can either by synchronous or asynchronous. + Asynchronous implementations should return + :class:`~google.cloud.ndb.tasklets.Future` instances whose eventual results + match the return value described for each method. Because coordinating with + the single threaded event model used by ``NDB`` can be tricky with remote + services, it's not recommended that casual users write asynchronous + implementations, as some specialized knowledge is required. + """ + + @abc.abstractmethod + def get(self, keys): + """Retrieve entities from the cache. + + Arguments: + keys (List[bytes]): The keys to get. + + Returns: + List[Union[bytes, None]]]: Serialized entities, or :data:`None`, + for each key. + """ + raise NotImplementedError + + @abc.abstractmethod + def set(self, items, expires=None): + """Store entities in the cache. + + Arguments: + items (Dict[bytes, Union[bytes, None]]): Mapping of keys to + serialized entities. + expires (Optional[float]): Number of seconds until value expires. + """ + raise NotImplementedError + + @abc.abstractmethod + def delete(self, keys): + """Remove entities from the cache. + + Arguments: + keys (List[bytes]): The keys to remove. + """ + raise NotImplementedError + + @abc.abstractmethod + def watch(self, keys): + """Begin an optimistic transaction for the given keys. + + A future call to :meth:`compare_and_swap` will only set values for keys + whose values haven't changed since the call to this method. + + Arguments: + keys (List[bytes]): The keys to watch. + """ + raise NotImplementedError + + @abc.abstractmethod + def compare_and_swap(self, items, expires=None): + """Like :meth:`set` but using an optimistic transaction. + + Only keys whose values haven't changed since a preceding call to + :meth:`watch` will be changed. + + Arguments: + items (Dict[bytes, Union[bytes, None]]): Mapping of keys to + serialized entities. + expires (Optional[float]): Number of seconds until value expires. + """ + raise NotImplementedError + + +class _InProcessGlobalCache(GlobalCache): + """Reference implementation of :class:`GlobalCache`. + + Not intended for production use. Uses a single process wide dictionary to + keep an in memory cache. For use in testing and to have an easily grokkable + reference implementation. Thread safety is potentially a little sketchy. + """ + + cache = {} + """Dict: The cache. + + Relies on atomicity of ``__setitem__`` for thread safety. See: + http://effbot.org/pyfaq/what-kinds-of-global-value-mutation-are-thread-safe.htm + """ + + def __init__(self): + self._watch_keys = {} + + def get(self, keys): + """Implements :meth:`GlobalCache.get`.""" + now = time.time() + results = [self.cache.get(key) for key in keys] + entity_pbs = [] + for result in results: + if result is not None: + entity_pb, expires = result + if expires and expires < now: + entity_pb = None + else: + entity_pb = None + + entity_pbs.append(entity_pb) + + return entity_pbs + + def set(self, items, expires=None): + """Implements :meth:`GlobalCache.set`.""" + if expires: + expires = time.time() + expires + + for key, value in items.items(): + self.cache[key] = (value, expires) # Supposedly threadsafe + + def delete(self, keys): + """Implements :meth:`GlobalCache.delete`.""" + for key in keys: + self.cache.pop(key, None) # Threadsafe? + + def watch(self, keys): + """Implements :meth:`GlobalCache.watch`.""" + for key in keys: + self._watch_keys[key] = self.cache.get(key) + + def compare_and_swap(self, items, expires=None): + """Implements :meth:`GlobalCache.compare_and_swap`.""" + if expires: + expires = time.time() + expires + + for key, new_value in items.items(): + watch_value = self._watch_keys.get(key) + current_value = self.cache.get(key) + if watch_value == current_value: + self.cache[key] = (new_value, expires) diff --git a/tests/conftest.py b/tests/conftest.py index 4aed6cfd..f4f9a5b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,9 +23,9 @@ from unittest import mock from google.cloud import environment_vars -from google.cloud.ndb import _cache from google.cloud.ndb import context as context_module from google.cloud.ndb import _eventloop +from google.cloud.ndb import global_cache as global_cache_module from google.cloud.ndb import model import pytest @@ -52,7 +52,7 @@ def reset_state(environ): yield model.Property._FIND_METHODS_CACHE.clear() model.Model._kind_map.clear() - _cache._InProcessGlobalCache.cache.clear() + global_cache_module._InProcessGlobalCache.cache.clear() @pytest.fixture @@ -103,8 +103,8 @@ def in_context(context): def global_cache(context): assert not context_module._state.context - global_cache = _cache._InProcessGlobalCache() - with context.new(global_cache=global_cache).use(): - yield global_cache + cache = global_cache_module._InProcessGlobalCache() + with context.new(global_cache=cache).use(): + yield cache assert not context_module._state.context diff --git a/tests/system/test_crud.py b/tests/system/test_crud.py index 27b497be..c25f7d8b 100644 --- a/tests/system/test_crud.py +++ b/tests/system/test_crud.py @@ -27,6 +27,7 @@ from google.cloud import ndb from google.cloud.ndb import _cache +from google.cloud.ndb import global_cache as global_cache_module from tests.system import KIND, eventually @@ -83,8 +84,8 @@ class SomeKind(ndb.Model): bar = ndb.StringProperty() baz = ndb.StringProperty() - global_cache = _cache._InProcessGlobalCache() - cache_dict = _cache._InProcessGlobalCache.cache + global_cache = global_cache_module._InProcessGlobalCache() + cache_dict = global_cache_module._InProcessGlobalCache.cache with client_context.new(global_cache=global_cache).use() as context: context.set_global_cache_policy(None) # Use default @@ -288,8 +289,8 @@ class SomeKind(ndb.Model): foo = ndb.IntegerProperty() bar = ndb.StringProperty() - global_cache = _cache._InProcessGlobalCache() - cache_dict = _cache._InProcessGlobalCache.cache + global_cache = global_cache_module._InProcessGlobalCache() + cache_dict = global_cache_module._InProcessGlobalCache.cache with client_context.new(global_cache=global_cache).use() as context: context.set_global_cache_policy(None) # Use default @@ -435,8 +436,8 @@ class SomeKind(ndb.Model): key = ndb.Key(KIND, entity_id) cache_key = _cache.global_cache_key(key._key) - global_cache = _cache._InProcessGlobalCache() - cache_dict = _cache._InProcessGlobalCache.cache + global_cache = global_cache_module._InProcessGlobalCache() + cache_dict = global_cache_module._InProcessGlobalCache.cache with client_context.new(global_cache=global_cache).use(): assert key.get().foo == 42 diff --git a/tests/unit/test__cache.py b/tests/unit/test__cache.py index ab47bfee..7d891bf5 100644 --- a/tests/unit/test__cache.py +++ b/tests/unit/test__cache.py @@ -56,131 +56,18 @@ def test_get_and_validate_miss(): cache.get_and_validate("nonexistent_key") -class TestGlobalCache: - def make_one(self): - class MockImpl(_cache.GlobalCache): - def get(self, keys): - return super(MockImpl, self).get(keys) - - def set(self, items, expires=None): - return super(MockImpl, self).set(items, expires=expires) - - def delete(self, keys): - return super(MockImpl, self).delete(keys) - - def watch(self, keys): - return super(MockImpl, self).watch(keys) - - def compare_and_swap(self, items, expires=None): - return super(MockImpl, self).compare_and_swap( - items, expires=expires - ) - - return MockImpl() - - def test_get(self): - cache = self.make_one() - with pytest.raises(NotImplementedError): - cache.get(b"foo") - - def test_set(self): - cache = self.make_one() - with pytest.raises(NotImplementedError): - cache.set({b"foo": "bar"}) - - def test_delete(self): - cache = self.make_one() - with pytest.raises(NotImplementedError): - cache.delete(b"foo") - - def test_watch(self): - cache = self.make_one() - with pytest.raises(NotImplementedError): - cache.watch(b"foo") - - def test_compare_and_swap(self): - cache = self.make_one() - with pytest.raises(NotImplementedError): - cache.compare_and_swap({b"foo": "bar"}) - - -class TestInProcessGlobalCache: - @staticmethod - def test_set_get_delete(): - cache = _cache._InProcessGlobalCache() - future = cache.set({b"one": b"foo", b"two": b"bar", b"three": b"baz"}) - assert future.result() is None - - future = cache.get([b"two", b"three", b"one"]) - assert future.result() == [b"bar", b"baz", b"foo"] - - cache = _cache._InProcessGlobalCache() - future = cache.get([b"two", b"three", b"one"]) - assert future.result() == [b"bar", b"baz", b"foo"] - - future = cache.delete([b"one", b"two", b"three"]) - assert future.result() is None - - future = cache.get([b"two", b"three", b"one"]) - assert future.result() == [None, None, None] - +class Test_GlobalCacheBatch: @staticmethod - @mock.patch("google.cloud.ndb._cache.time") - def test_set_get_delete_w_expires(time): - time.time.return_value = 0 - - cache = _cache._InProcessGlobalCache() - future = cache.set( - {b"one": b"foo", b"two": b"bar", b"three": b"baz"}, expires=5 - ) - assert future.result() is None - - future = cache.get([b"two", b"three", b"one"]) - assert future.result() == [b"bar", b"baz", b"foo"] - - time.time.return_value = 10 - future = cache.get([b"two", b"three", b"one"]) - assert future.result() == [None, None, None] - - @staticmethod - def test_watch_compare_and_swap(): - cache = _cache._InProcessGlobalCache() - future = cache.watch([b"one", b"two", b"three"]) - assert future.result() is None - - cache.cache[b"two"] = (b"hamburgers", None) - - future = cache.compare_and_swap( - {b"one": b"foo", b"two": b"bar", b"three": b"baz"} - ) - assert future.result() is None - - future = cache.get([b"one", b"two", b"three"]) - assert future.result() == [b"foo", b"hamburgers", b"baz"] + def test_make_call(): + batch = _cache._GlobalCacheBatch() + with pytest.raises(NotImplementedError): + batch.make_call() @staticmethod - @mock.patch("google.cloud.ndb._cache.time") - def test_watch_compare_and_swap_with_expires(time): - time.time.return_value = 0 - - cache = _cache._InProcessGlobalCache() - future = cache.watch([b"one", b"two", b"three"]) - assert future.result() is None - - cache.cache[b"two"] = (b"hamburgers", None) - - future = cache.compare_and_swap( - {b"one": b"foo", b"two": b"bar", b"three": b"baz"}, expires=5 - ) - assert future.result() is None - - future = cache.get([b"one", b"two", b"three"]) - assert future.result() == [b"foo", b"hamburgers", b"baz"] - - time.time.return_value = 10 - - future = cache.get([b"one", b"two", b"three"]) - assert future.result() == [None, b"hamburgers", None] + def test_future_info(): + batch = _cache._GlobalCacheBatch() + with pytest.raises(NotImplementedError): + batch.future_info(None) @mock.patch("google.cloud.ndb._cache._batch") diff --git a/tests/unit/test__datastore_api.py b/tests/unit/test__datastore_api.py index 1d0b6d62..ee077d93 100644 --- a/tests/unit/test__datastore_api.py +++ b/tests/unit/test__datastore_api.py @@ -216,7 +216,7 @@ class SomeKind(model.Model): future = _api.lookup(key._key, _options.ReadOptions()) assert future.result() == entity_pb - assert global_cache.get([cache_key]).result() == [cache_value] + assert global_cache.get([cache_key]) == [cache_value] @staticmethod @mock.patch("google.cloud.ndb._datastore_api._LookupBatch") @@ -259,7 +259,7 @@ class SomeKind(model.Model): future = _api.lookup(key._key, _options.ReadOptions()) assert future.result() == entity_pb - assert global_cache.get([cache_key]).result() == [_cache._LOCKED] + assert global_cache.get([cache_key]) == [_cache._LOCKED] @staticmethod @mock.patch("google.cloud.ndb._datastore_api._LookupBatch") @@ -276,7 +276,7 @@ class SomeKind(model.Model): future = _api.lookup(key._key, _options.ReadOptions()) assert future.result() is _api._NOT_FOUND - assert global_cache.get([cache_key]).result() == [_cache._LOCKED] + assert global_cache.get([cache_key]) == [_cache._LOCKED] class Test_LookupBatch: @@ -615,7 +615,7 @@ class SomeKind(model.Model): ) assert future.result() is None - assert global_cache.get([cache_key]).result() == [None] + assert global_cache.get([cache_key]) == [None] @staticmethod @mock.patch("google.cloud.ndb._datastore_api._NonTransactionalCommitBatch") @@ -636,7 +636,7 @@ class SomeKind(model.Model): ) assert future.result() == key._key - assert global_cache.get([cache_key]).result() == [None] + assert global_cache.get([cache_key]) == [None] class Test_delete: @@ -710,7 +710,7 @@ def test_cache_enabled(Batch, global_cache): future = _api.delete(key._key, _options.Options()) assert future.result() is None - assert global_cache.get([cache_key]).result() == [None] + assert global_cache.get([cache_key]) == [None] @staticmethod @mock.patch("google.cloud.ndb._datastore_api._NonTransactionalCommitBatch") @@ -726,7 +726,7 @@ def test_cache_disabled(Batch, global_cache): ) assert future.result() is None - assert global_cache.get([cache_key]).result() == [None] + assert global_cache.get([cache_key]) == [None] class Test_NonTransactionalCommitBatch: diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py index 4564a46f..7c9a7ee1 100644 --- a/tests/unit/test_context.py +++ b/tests/unit/test_context.py @@ -19,6 +19,7 @@ from google.cloud.ndb import context as context_module from google.cloud.ndb import _eventloop from google.cloud.ndb import exceptions +from google.cloud.ndb import global_cache from google.cloud.ndb import key as key_module from google.cloud.ndb import model from google.cloud.ndb import _options @@ -101,7 +102,9 @@ def test_clear_cache(self): assert not context.cache def test__clear_global_cache(self): - context = self._make_one(global_cache=_cache._InProcessGlobalCache()) + context = self._make_one( + global_cache=global_cache._InProcessGlobalCache() + ) with context.use(): key = key_module.Key("SomeKind", 1) cache_key = _cache.global_cache_key(key._key) @@ -113,7 +116,9 @@ def test__clear_global_cache(self): assert context.global_cache.cache == {"anotherkey": "otherdata"} def test__clear_global_cache_nothing_to_do(self): - context = self._make_one(global_cache=_cache._InProcessGlobalCache()) + context = self._make_one( + global_cache=global_cache._InProcessGlobalCache() + ) with context.use(): context.global_cache.cache["anotherkey"] = "otherdata" context._clear_global_cache().result() diff --git a/tests/unit/test_global_cache.py b/tests/unit/test_global_cache.py new file mode 100644 index 00000000..ffd6409a --- /dev/null +++ b/tests/unit/test_global_cache.py @@ -0,0 +1,146 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest + +from google.cloud.ndb import global_cache + + +class TestGlobalCache: + def make_one(self): + class MockImpl(global_cache.GlobalCache): + def get(self, keys): + return super(MockImpl, self).get(keys) + + def set(self, items, expires=None): + return super(MockImpl, self).set(items, expires=expires) + + def delete(self, keys): + return super(MockImpl, self).delete(keys) + + def watch(self, keys): + return super(MockImpl, self).watch(keys) + + def compare_and_swap(self, items, expires=None): + return super(MockImpl, self).compare_and_swap( + items, expires=expires + ) + + return MockImpl() + + def test_get(self): + cache = self.make_one() + with pytest.raises(NotImplementedError): + cache.get(b"foo") + + def test_set(self): + cache = self.make_one() + with pytest.raises(NotImplementedError): + cache.set({b"foo": "bar"}) + + def test_delete(self): + cache = self.make_one() + with pytest.raises(NotImplementedError): + cache.delete(b"foo") + + def test_watch(self): + cache = self.make_one() + with pytest.raises(NotImplementedError): + cache.watch(b"foo") + + def test_compare_and_swap(self): + cache = self.make_one() + with pytest.raises(NotImplementedError): + cache.compare_and_swap({b"foo": "bar"}) + + +class TestInProcessGlobalCache: + @staticmethod + def test_set_get_delete(): + cache = global_cache._InProcessGlobalCache() + result = cache.set({b"one": b"foo", b"two": b"bar", b"three": b"baz"}) + assert result is None + + result = cache.get([b"two", b"three", b"one"]) + assert result == [b"bar", b"baz", b"foo"] + + cache = global_cache._InProcessGlobalCache() + result = cache.get([b"two", b"three", b"one"]) + assert result == [b"bar", b"baz", b"foo"] + + result = cache.delete([b"one", b"two", b"three"]) + assert result is None + + result = cache.get([b"two", b"three", b"one"]) + assert result == [None, None, None] + + @staticmethod + @mock.patch("google.cloud.ndb.global_cache.time") + def test_set_get_delete_w_expires(time): + time.time.return_value = 0 + + cache = global_cache._InProcessGlobalCache() + result = cache.set( + {b"one": b"foo", b"two": b"bar", b"three": b"baz"}, expires=5 + ) + assert result is None + + result = cache.get([b"two", b"three", b"one"]) + assert result == [b"bar", b"baz", b"foo"] + + time.time.return_value = 10 + result = cache.get([b"two", b"three", b"one"]) + assert result == [None, None, None] + + @staticmethod + def test_watch_compare_and_swap(): + cache = global_cache._InProcessGlobalCache() + result = cache.watch([b"one", b"two", b"three"]) + assert result is None + + cache.cache[b"two"] = (b"hamburgers", None) + + result = cache.compare_and_swap( + {b"one": b"foo", b"two": b"bar", b"three": b"baz"} + ) + assert result is None + + result = cache.get([b"one", b"two", b"three"]) + assert result == [b"foo", b"hamburgers", b"baz"] + + @staticmethod + @mock.patch("google.cloud.ndb.global_cache.time") + def test_watch_compare_and_swap_with_expires(time): + time.time.return_value = 0 + + cache = global_cache._InProcessGlobalCache() + result = cache.watch([b"one", b"two", b"three"]) + assert result is None + + cache.cache[b"two"] = (b"hamburgers", None) + + result = cache.compare_and_swap( + {b"one": b"foo", b"two": b"bar", b"three": b"baz"}, expires=5 + ) + assert result is None + + result = cache.get([b"one", b"two", b"three"]) + assert result == [b"foo", b"hamburgers", b"baz"] + + time.time.return_value = 10 + + result = cache.get([b"one", b"two", b"three"]) + assert result == [None, b"hamburgers", None]