From 7bd719a22b77fe81f11c8bc8fe5e7402bdfc11de Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Sat, 18 Feb 2023 07:26:44 -0800 Subject: [PATCH 01/34] Move namespace logic to BaseCache.build_key() --- aiocache/base.py | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/aiocache/base.py b/aiocache/base.py index 54e9d18c..c7fad3c3 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -114,7 +114,7 @@ def __init__( self.timeout = float(timeout) if timeout is not None else timeout self.namespace = namespace self.ttl = float(ttl) if ttl is not None else ttl - self.build_key = key_builder or self._build_key + self._build_key = key_builder or (lambda key, ns: "{}{}".format(ns or "", key)) self._serializer = None self.serializer = serializer or serializers.StringSerializer() @@ -163,8 +163,7 @@ async def add(self, key, value, ttl=SENTINEL, dumps_fn=None, namespace=None, _co """ start = time.monotonic() dumps = dumps_fn or self._serializer.dumps - ns = namespace if namespace is not None else self.namespace - ns_key = self.build_key(key, namespace=ns) + ns_key = self.build_key(key, namespace=namespace) await self._add(ns_key, dumps(value), ttl=self._get_ttl(ttl), _conn=_conn) @@ -193,8 +192,7 @@ async def get(self, key, default=None, loads_fn=None, namespace=None, _conn=None """ start = time.monotonic() loads = loads_fn or self._serializer.loads - ns = namespace if namespace is not None else self.namespace - ns_key = self.build_key(key, namespace=ns) + ns_key = self.build_key(key, namespace=namespace) value = loads(await self._get(ns_key, encoding=self.serializer.encoding, _conn=_conn)) @@ -225,9 +223,8 @@ async def multi_get(self, keys, loads_fn=None, namespace=None, _conn=None): """ start = time.monotonic() loads = loads_fn or self._serializer.loads - ns = namespace if namespace is not None else self.namespace - ns_keys = [self.build_key(key, namespace=ns) for key in keys] + ns_keys = [self.build_key(key, namespace=namespace) for key in keys] values = [ loads(value) for value in await self._multi_get( @@ -270,8 +267,7 @@ async def set( """ start = time.monotonic() dumps = dumps_fn or self._serializer.dumps - ns = namespace if namespace is not None else self.namespace - ns_key = self.build_key(key, namespace=ns) + ns_key = self.build_key(key, namespace=namespace) res = await self._set( ns_key, dumps(value), ttl=self._get_ttl(ttl), _cas_token=_cas_token, _conn=_conn @@ -304,11 +300,10 @@ async def multi_set(self, pairs, ttl=SENTINEL, dumps_fn=None, namespace=None, _c """ start = time.monotonic() dumps = dumps_fn or self._serializer.dumps - ns = namespace if namespace is not None else self.namespace tmp_pairs = [] for key, value in pairs: - tmp_pairs.append((self.build_key(key, namespace=ns), dumps(value))) + tmp_pairs.append((self.build_key(key, namespace=namespace), dumps(value))) await self._multi_set(tmp_pairs, ttl=self._get_ttl(ttl), _conn=_conn) @@ -339,8 +334,7 @@ async def delete(self, key, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - ns = namespace if namespace is not None else self.namespace - ns_key = self.build_key(key, namespace=ns) + ns_key = self.build_key(key, namespace=namespace) ret = await self._delete(ns_key, _conn=_conn) logger.debug("DELETE %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret @@ -364,8 +358,7 @@ async def exists(self, key, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - ns = namespace if namespace is not None else self.namespace - ns_key = self.build_key(key, namespace=ns) + ns_key = self.build_key(key, namespace=namespace) ret = await self._exists(ns_key, _conn=_conn) logger.debug("EXISTS %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret @@ -392,8 +385,7 @@ async def increment(self, key, delta=1, namespace=None, _conn=None): :raises: :class:`TypeError` if value is not incrementable """ start = time.monotonic() - ns = namespace if namespace is not None else self.namespace - ns_key = self.build_key(key, namespace=ns) + ns_key = self.build_key(key, namespace=namespace) ret = await self._increment(ns_key, delta, _conn=_conn) logger.debug("INCREMENT %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret @@ -418,8 +410,7 @@ async def expire(self, key, ttl, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - ns = namespace if namespace is not None else self.namespace - ns_key = self.build_key(key, namespace=ns) + ns_key = self.build_key(key, namespace=namespace) ret = await self._expire(ns_key, ttl, _conn=_conn) logger.debug("EXPIRE %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret @@ -498,12 +489,9 @@ async def close(self, *args, _conn=None, **kwargs): async def _close(self, *args, **kwargs): pass - def _build_key(self, key, namespace=None): - if namespace is not None: - return "{}{}".format(namespace, _ensure_key(key)) - if self.namespace is not None: - return "{}{}".format(self.namespace, _ensure_key(key)) - return key + def build_key(self, key, namespace=None): + ns = namespace if namespace is not None else self.namespace + return self._build_key(_ensure_key(key), namespace=ns) def _get_ttl(self, ttl): return ttl if ttl is not SENTINEL else self.ttl From 0eeab3e004b47457a1869433071ecb74e62bebb4 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Sat, 18 Feb 2023 08:22:34 -0800 Subject: [PATCH 02/34] BaseCache._build_key() needs keyword param Replace lambda function with member function for self._build_key fallback Allows using 'namespace' as named parameter, as expected --- aiocache/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aiocache/base.py b/aiocache/base.py index c7fad3c3..1135d6d6 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -114,7 +114,7 @@ def __init__( self.timeout = float(timeout) if timeout is not None else timeout self.namespace = namespace self.ttl = float(ttl) if ttl is not None else ttl - self._build_key = key_builder or (lambda key, ns: "{}{}".format(ns or "", key)) + self._build_key = key_builder or self._build_key_default self._serializer = None self.serializer = serializer or serializers.StringSerializer() @@ -492,6 +492,9 @@ async def _close(self, *args, **kwargs): def build_key(self, key, namespace=None): ns = namespace if namespace is not None else self.namespace return self._build_key(_ensure_key(key), namespace=ns) + + def _build_key_default(self, key, namespace=None): + return "{}{}".format(namespace or "", key) def _get_ttl(self, ttl): return ttl if ttl is not SENTINEL else self.ttl From 25943bdfad72988bada5194b238e7cb3a23b9fbf Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Sun, 19 Feb 2023 07:27:14 -0800 Subject: [PATCH 03/34] ut/test_base/alt_base_cache tests use clearer default namespace Documenting for posterity how this test could have been clearer for previously expected functionality. --- tests/ut/test_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index 569b5245..605ce09d 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -218,7 +218,7 @@ def test_alt_build_key(self): def alt_base_cache(self, init_namespace="test"): """Custom key_builder for cache""" def build_key(key, namespace=None): - ns = namespace if namespace is not None else "" + ns = namespace if namespace is not None else "alt" sep = ":" if namespace else "" return f"{ns}{sep}{_ensure_key(key)}" @@ -227,7 +227,7 @@ def build_key(key, namespace=None): @pytest.mark.parametrize( "namespace, expected", - ([None, _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "alt" + _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) def test_alt_build_key_override_namespace(self, alt_base_cache, namespace, expected): """Custom key_builder overrides namespace of cache""" @@ -236,7 +236,7 @@ def test_alt_build_key_override_namespace(self, alt_base_cache, namespace, expec @pytest.mark.parametrize( "init_namespace, expected", - ([None, _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["test", "test:" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "alt" + _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["test", "test:" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) async def test_alt_build_key_default_namespace( self, init_namespace, alt_base_cache, expected): From fcd7afc4514e2b6ab06c88e9f12e9d468de123e8 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Sun, 19 Feb 2023 07:39:32 -0800 Subject: [PATCH 04/34] Update ut/test_base/alt_base_cache tests with new build_key() namespace logic --- tests/ut/test_base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index 605ce09d..5db96539 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -218,16 +218,15 @@ def test_alt_build_key(self): def alt_base_cache(self, init_namespace="test"): """Custom key_builder for cache""" def build_key(key, namespace=None): - ns = namespace if namespace is not None else "alt" sep = ":" if namespace else "" - return f"{ns}{sep}{_ensure_key(key)}" + return f'{namespace or ""}{sep}{_ensure_key(key)}' cache = BaseCache(key_builder=build_key, namespace=init_namespace) return cache @pytest.mark.parametrize( "namespace, expected", - ([None, "alt" + _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "test:" + _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) def test_alt_build_key_override_namespace(self, alt_base_cache, namespace, expected): """Custom key_builder overrides namespace of cache""" @@ -236,7 +235,7 @@ def test_alt_build_key_override_namespace(self, alt_base_cache, namespace, expec @pytest.mark.parametrize( "init_namespace, expected", - ([None, "alt" + _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["test", "test:" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["test", "test:" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) async def test_alt_build_key_default_namespace( self, init_namespace, alt_base_cache, expected): From 139e0a4d712cefc816c5c2c499f62dc97c870078 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Sun, 19 Feb 2023 09:10:39 -0800 Subject: [PATCH 05/34] Update redis namespaced key for new build_key() compatibility --- aiocache/backends/redis.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/aiocache/backends/redis.py b/aiocache/backends/redis.py index 407efec1..53be9d90 100644 --- a/aiocache/backends/redis.py +++ b/aiocache/backends/redis.py @@ -4,7 +4,7 @@ import redis.asyncio as redis from redis.exceptions import ResponseError as IncrbyException -from aiocache.base import BaseCache, _ensure_key +from aiocache.base import BaseCache from aiocache.serializers import JsonSerializer @@ -218,14 +218,8 @@ def parse_uri_path(cls, path): options["db"] = db return options - def _build_key(self, key, namespace=None): - if namespace is not None: - return "{}{}{}".format( - namespace, ":" if namespace else "", _ensure_key(key)) - if self.namespace is not None: - return "{}{}{}".format( - self.namespace, ":" if self.namespace else "", _ensure_key(key)) - return key + def _build_key_default(self, key, namespace=None): + return "{}{}{}".format(namespace or "", ":" if namespace else "", key) def __repr__(self): # pragma: no cover return "RedisCache ({}:{})".format(self.endpoint, self.port) From eebe2998f5f81a2d91c003d21ffc8e9d7881a90f Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Sun, 19 Feb 2023 09:24:07 -0800 Subject: [PATCH 06/34] Update memcached key encoding for new build_key() compatibility --- aiocache/backends/memcached.py | 4 ++-- aiocache/base.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aiocache/backends/memcached.py b/aiocache/backends/memcached.py index 8abf56f7..b86f13e4 100644 --- a/aiocache/backends/memcached.py +++ b/aiocache/backends/memcached.py @@ -147,8 +147,8 @@ def __init__(self, serializer=None, **kwargs): def parse_uri_path(cls, path): return {} - def _build_key(self, key, namespace=None): - ns_key = super()._build_key(key, namespace=namespace).replace(" ", "_") + def build_key(self, key, namespace=None): + ns_key = super().build_key(key, namespace=namespace).replace(" ", "_") return str.encode(ns_key) def __repr__(self): # pragma: no cover diff --git a/aiocache/base.py b/aiocache/base.py index 1135d6d6..41dbe736 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -492,7 +492,7 @@ async def _close(self, *args, **kwargs): def build_key(self, key, namespace=None): ns = namespace if namespace is not None else self.namespace return self._build_key(_ensure_key(key), namespace=ns) - + def _build_key_default(self, key, namespace=None): return "{}{}".format(namespace or "", key) From 6622f57e9dfceb072d82e6d52c77398ce8c63d6b Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Sun, 19 Feb 2023 09:33:48 -0800 Subject: [PATCH 07/34] test/ut/test_base.py now calls build_key(), rather than _build_key() --- tests/acceptance/test_lock.py | 2 +- tests/ut/test_base.py | 24 ++++++++++++------------ tests/ut/test_lock.py | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/acceptance/test_lock.py b/tests/acceptance/test_lock.py index 665fea90..f151d618 100644 --- a/tests/acceptance/test_lock.py +++ b/tests/acceptance/test_lock.py @@ -200,7 +200,7 @@ def lock(self, cache): async def test_acquire(self, cache, lock): await cache.set(Keys.KEY, "value") async with lock: - assert lock._token == await cache._gets(cache._build_key(Keys.KEY)) + assert lock._token == await cache._gets(cache.build_key(Keys.KEY)) async def test_release_does_nothing(self, lock): assert await lock.__aexit__("exc_type", "exc_value", "traceback") is None diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index 5db96539..47bdf017 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -428,7 +428,7 @@ async def test_get(self, mock_base_cache): await mock_base_cache.get(Keys.KEY) mock_base_cache._get.assert_called_with( - mock_base_cache._build_key(Keys.KEY), encoding=ANY, _conn=ANY + mock_base_cache.build_key(Keys.KEY), encoding=ANY, _conn=ANY ) assert mock_base_cache.plugins[0].pre_get.call_count == 1 assert mock_base_cache.plugins[0].post_get.call_count == 1 @@ -453,7 +453,7 @@ async def test_set(self, mock_base_cache): await mock_base_cache.set(Keys.KEY, "value", ttl=2) mock_base_cache._set.assert_called_with( - mock_base_cache._build_key(Keys.KEY), ANY, ttl=2, _cas_token=None, _conn=ANY + mock_base_cache.build_key(Keys.KEY), ANY, ttl=2, _cas_token=None, _conn=ANY ) assert mock_base_cache.plugins[0].pre_set.call_count == 1 assert mock_base_cache.plugins[0].post_set.call_count == 1 @@ -468,7 +468,7 @@ async def test_add(self, mock_base_cache): mock_base_cache._exists = AsyncMock(return_value=False) await mock_base_cache.add(Keys.KEY, "value", ttl=2) - key = mock_base_cache._build_key(Keys.KEY) + key = mock_base_cache.build_key(Keys.KEY) mock_base_cache._add.assert_called_with(key, ANY, ttl=2, _conn=ANY) assert mock_base_cache.plugins[0].pre_add.call_count == 1 assert mock_base_cache.plugins[0].post_add.call_count == 1 @@ -483,7 +483,7 @@ async def test_mget(self, mock_base_cache): await mock_base_cache.multi_get([Keys.KEY, Keys.KEY_1]) mock_base_cache._multi_get.assert_called_with( - [mock_base_cache._build_key(Keys.KEY), mock_base_cache._build_key(Keys.KEY_1)], + [mock_base_cache.build_key(Keys.KEY), mock_base_cache.build_key(Keys.KEY_1)], encoding=ANY, _conn=ANY, ) @@ -499,8 +499,8 @@ async def test_mget_timeouts(self, mock_base_cache): async def test_mset(self, mock_base_cache): await mock_base_cache.multi_set([[Keys.KEY, "value"], [Keys.KEY_1, "value1"]], ttl=2) - key = mock_base_cache._build_key(Keys.KEY) - key1 = mock_base_cache._build_key(Keys.KEY_1) + key = mock_base_cache.build_key(Keys.KEY) + key1 = mock_base_cache.build_key(Keys.KEY_1) mock_base_cache._multi_set.assert_called_with( [(key, ANY), (key1, ANY)], ttl=2, _conn=ANY) assert mock_base_cache.plugins[0].pre_multi_set.call_count == 1 @@ -515,7 +515,7 @@ async def test_mset_timeouts(self, mock_base_cache): async def test_exists(self, mock_base_cache): await mock_base_cache.exists(Keys.KEY) - mock_base_cache._exists.assert_called_with(mock_base_cache._build_key(Keys.KEY), _conn=ANY) + mock_base_cache._exists.assert_called_with(mock_base_cache.build_key(Keys.KEY), _conn=ANY) assert mock_base_cache.plugins[0].pre_exists.call_count == 1 assert mock_base_cache.plugins[0].post_exists.call_count == 1 @@ -528,7 +528,7 @@ async def test_exists_timeouts(self, mock_base_cache): async def test_increment(self, mock_base_cache): await mock_base_cache.increment(Keys.KEY, 2) - key = mock_base_cache._build_key(Keys.KEY) + key = mock_base_cache.build_key(Keys.KEY) mock_base_cache._increment.assert_called_with(key, 2, _conn=ANY) assert mock_base_cache.plugins[0].pre_increment.call_count == 1 assert mock_base_cache.plugins[0].post_increment.call_count == 1 @@ -542,7 +542,7 @@ async def test_increment_timeouts(self, mock_base_cache): async def test_delete(self, mock_base_cache): await mock_base_cache.delete(Keys.KEY) - mock_base_cache._delete.assert_called_with(mock_base_cache._build_key(Keys.KEY), _conn=ANY) + mock_base_cache._delete.assert_called_with(mock_base_cache.build_key(Keys.KEY), _conn=ANY) assert mock_base_cache.plugins[0].pre_delete.call_count == 1 assert mock_base_cache.plugins[0].post_delete.call_count == 1 @@ -554,7 +554,7 @@ async def test_delete_timeouts(self, mock_base_cache): async def test_expire(self, mock_base_cache): await mock_base_cache.expire(Keys.KEY, 1) - key = mock_base_cache._build_key(Keys.KEY) + key = mock_base_cache.build_key(Keys.KEY) mock_base_cache._expire.assert_called_with(key, 1, _conn=ANY) assert mock_base_cache.plugins[0].pre_expire.call_count == 1 assert mock_base_cache.plugins[0].post_expire.call_count == 1 @@ -567,7 +567,7 @@ async def test_expire_timeouts(self, mock_base_cache): async def test_clear(self, mock_base_cache): await mock_base_cache.clear(Keys.KEY) - mock_base_cache._clear.assert_called_with(mock_base_cache._build_key(Keys.KEY), _conn=ANY) + mock_base_cache._clear.assert_called_with(mock_base_cache.build_key(Keys.KEY), _conn=ANY) assert mock_base_cache.plugins[0].pre_clear.call_count == 1 assert mock_base_cache.plugins[0].post_clear.call_count == 1 @@ -580,7 +580,7 @@ async def test_clear_timeouts(self, mock_base_cache): async def test_raw(self, mock_base_cache): await mock_base_cache.raw("get", Keys.KEY) mock_base_cache._raw.assert_called_with( - "get", mock_base_cache._build_key(Keys.KEY), encoding=ANY, _conn=ANY + "get", mock_base_cache.build_key(Keys.KEY), encoding=ANY, _conn=ANY ) assert mock_base_cache.plugins[0].pre_raw.call_count == 1 assert mock_base_cache.plugins[0].post_raw.call_count == 1 diff --git a/tests/ut/test_lock.py b/tests/ut/test_lock.py index 1bac2bff..a06ea567 100644 --- a/tests/ut/test_lock.py +++ b/tests/ut/test_lock.py @@ -86,7 +86,7 @@ def test_init(self, mock_base_cache, lock): assert lock.client == mock_base_cache assert lock._token is None assert lock.key == Keys.KEY - assert lock.ns_key == mock_base_cache._build_key(Keys.KEY) + assert lock.ns_key == mock_base_cache.build_key(Keys.KEY) async def test_aenter_returns_lock(self, lock): assert await lock.__aenter__() is lock From 54ce48df3c03243b3367201fa6715fa7204ad586 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Sun, 19 Feb 2023 09:46:09 -0800 Subject: [PATCH 08/34] Move Enum _ensure_key() logic into BaseCache:build_key() Tests now defines this functionality in tests/utils.py --- aiocache/base.py | 10 ++-------- tests/acceptance/test_decorators.py | 9 ++++----- tests/ut/backends/test_memcached.py | 6 +++--- tests/ut/backends/test_redis.py | 6 +++--- tests/ut/test_base.py | 12 ++++++------ tests/utils.py | 7 +++++++ 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/aiocache/base.py b/aiocache/base.py index 41dbe736..c73dbe83 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -490,8 +490,9 @@ async def _close(self, *args, **kwargs): pass def build_key(self, key, namespace=None): + key_name = key.value if isinstance(key, Enum) else key ns = namespace if namespace is not None else self.namespace - return self._build_key(_ensure_key(key), namespace=ns) + return self._build_key(key_name, namespace=ns) def _build_key_default(self, key, namespace=None): return "{}{}".format(namespace or "", key) @@ -541,12 +542,5 @@ async def _do_inject_conn(self, *args, **kwargs): return _do_inject_conn -def _ensure_key(key): - if isinstance(key, Enum): - return key.value - else: - return key - - for cmd in API.CMDS: setattr(_Conn, cmd.__name__, _Conn._inject_conn(cmd.__name__)) diff --git a/tests/acceptance/test_decorators.py b/tests/acceptance/test_decorators.py index f9608e56..ad99aca7 100644 --- a/tests/acceptance/test_decorators.py +++ b/tests/acceptance/test_decorators.py @@ -5,8 +5,7 @@ import pytest from aiocache import cached, cached_stampede, multi_cached -from aiocache.base import _ensure_key -from ..utils import Keys +from ..utils import Keys, ensure_key async def return_dict(keys=None): @@ -164,15 +163,15 @@ async def fn(keys): async def test_multi_cached_key_builder(self, cache): def build_key(key, f, self, keys, market="ES"): - return "{}_{}_{}".format(f.__name__, _ensure_key(key), market) + return "{}_{}_{}".format(f.__name__, ensure_key(key), market) @multi_cached(keys_from_attr="keys", key_builder=build_key) async def fn(self, keys, market="ES"): return {Keys.KEY: 1, Keys.KEY_1: 2} await fn("self", keys=[Keys.KEY, Keys.KEY_1]) - assert await cache.exists("fn_" + _ensure_key(Keys.KEY) + "_ES") is True - assert await cache.exists("fn_" + _ensure_key(Keys.KEY_1) + "_ES") is True + assert await cache.exists("fn_" + ensure_key(Keys.KEY) + "_ES") is True + assert await cache.exists("fn_" + ensure_key(Keys.KEY_1) + "_ES") is True async def test_multi_cached_skip_keys(self, cache): @multi_cached(keys_from_attr="keys", skip_cache_func=lambda _, v: v is None) diff --git a/tests/ut/backends/test_memcached.py b/tests/ut/backends/test_memcached.py index 7e011618..52a764b7 100644 --- a/tests/ut/backends/test_memcached.py +++ b/tests/ut/backends/test_memcached.py @@ -4,9 +4,9 @@ import pytest from aiocache.backends.memcached import MemcachedBackend, MemcachedCache -from aiocache.base import BaseCache, _ensure_key +from aiocache.base import BaseCache from aiocache.serializers import JsonSerializer -from ...utils import Keys +from ...utils import Keys, ensure_key @pytest.fixture @@ -249,7 +249,7 @@ def test_parse_uri_path(self): @pytest.mark.parametrize( "namespace, expected", - ([None, "test" + _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["my_ns", "my_ns" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "test" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) def test_build_key_bytes(self, set_test_namespace, memcached_cache, namespace, expected): assert memcached_cache.build_key(Keys.KEY, namespace=namespace) == expected.encode() diff --git a/tests/ut/backends/test_redis.py b/tests/ut/backends/test_redis.py index 2470584a..8f2b6337 100644 --- a/tests/ut/backends/test_redis.py +++ b/tests/ut/backends/test_redis.py @@ -5,9 +5,9 @@ from redis.exceptions import ResponseError from aiocache.backends.redis import RedisBackend, RedisCache -from aiocache.base import BaseCache, _ensure_key +from aiocache.base import BaseCache from aiocache.serializers import JsonSerializer -from ...utils import Keys +from ...utils import Keys, ensure_key @pytest.fixture @@ -253,7 +253,7 @@ def test_parse_uri_path(self, path, expected): @pytest.mark.parametrize( "namespace, expected", - ([None, "test:" + _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + _ensure_key(Keys.KEY)]), # noqa: B950 + ([None, "test:" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_build_key_double_dot(self, set_test_namespace, redis_cache, namespace, expected): assert redis_cache.build_key(Keys.KEY, namespace=namespace) == expected diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index 47bdf017..bd929817 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -4,8 +4,8 @@ import pytest -from aiocache.base import API, BaseCache, _Conn, _ensure_key -from ..utils import Keys +from aiocache.base import API, BaseCache, _Conn +from ..utils import Keys, ensure_key class TestAPI: @@ -205,7 +205,7 @@ def set_test_namespace(self, base_cache): @pytest.mark.parametrize( "namespace, expected", - ([None, "test" + _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["my_ns", "my_ns" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "test" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) def test_build_key(self, set_test_namespace, base_cache, namespace, expected): assert base_cache.build_key(Keys.KEY, namespace=namespace) == expected @@ -219,14 +219,14 @@ def alt_base_cache(self, init_namespace="test"): """Custom key_builder for cache""" def build_key(key, namespace=None): sep = ":" if namespace else "" - return f'{namespace or ""}{sep}{_ensure_key(key)}' + return f'{namespace or ""}{sep}{ensure_key(key)}' cache = BaseCache(key_builder=build_key, namespace=init_namespace) return cache @pytest.mark.parametrize( "namespace, expected", - ([None, "test:" + _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "test:" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) def test_alt_build_key_override_namespace(self, alt_base_cache, namespace, expected): """Custom key_builder overrides namespace of cache""" @@ -235,7 +235,7 @@ def test_alt_build_key_override_namespace(self, alt_base_cache, namespace, expec @pytest.mark.parametrize( "init_namespace, expected", - ([None, _ensure_key(Keys.KEY)], ["", _ensure_key(Keys.KEY)], ["test", "test:" + _ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["test", "test:" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) async def test_alt_build_key_default_namespace( self, init_namespace, alt_base_cache, expected): diff --git a/tests/utils.py b/tests/utils.py index ab884c7f..a6f622fe 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -7,3 +7,10 @@ class Keys(str, Enum): KEY_LOCK = Keys.KEY + "-lock" + + +def ensure_key(key): + if isinstance(key, Enum): + return key.value + else: + return key From b2e58d79dc3ada570716364c8281077513b52566 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Sun, 19 Feb 2023 10:05:21 -0800 Subject: [PATCH 09/34] Update changelog with revisied cache build_key() scheme --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 0f539e4a..8907f575 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -15,6 +15,10 @@ There are a number of backwards-incompatible changes. These points should help w * The ``key`` parameter has been removed from the ``cached`` decorator. The behaviour can be easily reimplemented with ``key_builder=lambda *a, **kw: "foo"`` * When using the ``key_builder`` parameter in ``@multicached``, the function will now return the original, unmodified keys, only using the transformed keys in the cache (this has always been the documented behaviour, but not the implemented behaviour). * ``BaseSerializer`` is now an ``ABC``, so cannot be instantiated directly. +* The logic for encoding a cache key and selecting the key's namespace is now encapsulated in +the ``build_key(key, namespace)`` member of BaseCache (and its backend subclasses). Now +creating a cache with a custom ``key_builder`` argument simply requires that function to +return any string mapping from the ``key`` and optional ``namespace`` parameters. 0.12.0 (2023-01-13) From 223b537137b23a89066f689243a1a41b41d6fce9 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Sun, 19 Feb 2023 10:09:17 -0800 Subject: [PATCH 10/34] Clarify that output of BaseCache key_builder arg should be a string --- aiocache/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiocache/base.py b/aiocache/base.py index c73dbe83..96b50285 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -99,7 +99,7 @@ class BaseCache: :param namespace: string to use as default prefix for the key used in all operations of the backend. Default is None :param key_builder: alternative callable to build the key. Receives the key and the namespace - as params and should return something that can be used as key by the underlying backend. + as params and should return a string that can be used as key by the underlying backend. :param timeout: int or float in seconds specifying maximum timeout for the operations to last. By default its 5. Use 0 or None if you want to disable it. :param ttl: int the expiration time in seconds to use as a default in all operations of From e8d60373b15b34737fb57568346cb5e104226432 Mon Sep 17 00:00:00 2001 From: pshafer-als <76011594+pshafer-als@users.noreply.github.com> Date: Mon, 20 Feb 2023 09:56:31 -0800 Subject: [PATCH 11/34] Clarify 'key_builder' parameter docstring in 'BaseCache.__init__()' Co-authored-by: Sam Bull --- aiocache/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiocache/base.py b/aiocache/base.py index 96b50285..5be2f323 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -99,7 +99,7 @@ class BaseCache: :param namespace: string to use as default prefix for the key used in all operations of the backend. Default is None :param key_builder: alternative callable to build the key. Receives the key and the namespace - as params and should return a string that can be used as key by the underlying backend. + as params and should return a string that can be used as a key by the underlying backend. :param timeout: int or float in seconds specifying maximum timeout for the operations to last. By default its 5. Use 0 or None if you want to disable it. :param ttl: int the expiration time in seconds to use as a default in all operations of From db9426196d0a4b55a78411c8b96949af645a557b Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Mon, 20 Feb 2023 17:56:54 -0800 Subject: [PATCH 12/34] Ensure that cache namespace is a string Include type annotations in updated functions. --- aiocache/backends/memcached.py | 5 +++-- aiocache/backends/memory.py | 2 +- aiocache/backends/redis.py | 24 +++++++++++++++------ aiocache/base.py | 39 +++++++++++++++++++++------------- aiocache/decorators.py | 6 +++--- 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/aiocache/backends/memcached.py b/aiocache/backends/memcached.py index b86f13e4..6ea5378f 100644 --- a/aiocache/backends/memcached.py +++ b/aiocache/backends/memcached.py @@ -1,4 +1,5 @@ import asyncio +from typing import Optional import aiomcache @@ -130,7 +131,7 @@ class MemcachedCache(MemcachedBackend): :param serializer: obj derived from :class:`aiocache.serializers.BaseSerializer`. :param plugins: list of :class:`aiocache.plugins.BasePlugin` derived classes. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None + the backend. Default is an empty string, "". :param timeout: int or float in seconds specifying maximum timeout for the operations to last. By default its 5. :param endpoint: str with the endpoint to connect to. Default is 127.0.0.1. @@ -147,7 +148,7 @@ def __init__(self, serializer=None, **kwargs): def parse_uri_path(cls, path): return {} - def build_key(self, key, namespace=None): + def build_key(self, key: str, namespace: Optional[str] = None) -> str: ns_key = super().build_key(key, namespace=namespace).replace(" ", "_") return str.encode(ns_key) diff --git a/aiocache/backends/memory.py b/aiocache/backends/memory.py index 7d75d7ee..2936086a 100644 --- a/aiocache/backends/memory.py +++ b/aiocache/backends/memory.py @@ -118,7 +118,7 @@ class SimpleMemoryCache(SimpleMemoryBackend): :param serializer: obj derived from :class:`aiocache.serializers.BaseSerializer`. :param plugins: list of :class:`aiocache.plugins.BasePlugin` derived classes. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None. + the backend. Default is an empty string, "". :param timeout: int or float in seconds specifying maximum timeout for the operations to last. By default its 5. """ diff --git a/aiocache/backends/redis.py b/aiocache/backends/redis.py index 372f4eeb..05539e27 100644 --- a/aiocache/backends/redis.py +++ b/aiocache/backends/redis.py @@ -1,11 +1,12 @@ import itertools import warnings +from typing import Callable, Optional import redis.asyncio as redis from redis.exceptions import ResponseError as IncrbyException from aiocache.base import BaseCache -from aiocache.serializers import JsonSerializer +from aiocache.serializers import BaseSerializer, JsonSerializer _NOT_SET = object() @@ -186,7 +187,7 @@ class RedisCache(RedisBackend): :param serializer: obj derived from :class:`aiocache.serializers.BaseSerializer`. :param plugins: list of :class:`aiocache.plugins.BasePlugin` derived classes. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None. + the backend. Default is an empty string, "". :param timeout: int or float in seconds specifying maximum timeout for the operations to last. By default its 5. :param endpoint: str with the endpoint to connect to. Default is "127.0.0.1". @@ -199,8 +200,19 @@ class RedisCache(RedisBackend): NAME = "redis" - def __init__(self, serializer=None, **kwargs): - super().__init__(serializer=serializer or JsonSerializer(), **kwargs) + def __init__( + self, + serializer: Optional[BaseSerializer] = None, + namespace: str = "", + key_builder: Optional[Callable[[str, str], str]] = None, + **kwargs, + ): + super().__init__( + serializer=serializer or JsonSerializer(), + namespace=namespace, + key_builder=key_builder or ( + lambda key, namespace="": f"{namespace}:{key}" if namespace else key), + **kwargs) @classmethod def parse_uri_path(cls, path): @@ -218,8 +230,8 @@ def parse_uri_path(cls, path): options["db"] = db return options - def _build_key_default(self, key, namespace=None): - return "{}{}{}".format(namespace or "", ":" if namespace else "", key) + def _build_key_default(self, key: str, namespace: str = "") -> str: + return f"{namespace}:{key}" if namespace else key def __repr__(self): # pragma: no cover return "RedisCache ({}:{})".format(self.endpoint, self.port) diff --git a/aiocache/base.py b/aiocache/base.py index 5be2f323..c4243350 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -5,9 +5,10 @@ import time from enum import Enum from types import TracebackType -from typing import Callable, Optional, Set, Type +from typing import Any, Callable, List, Optional, Set, Type -from aiocache import serializers +# from aiocache.plugins import BasePlugin +from aiocache.serializers import BaseSerializer, StringSerializer logger = logging.getLogger(__name__) @@ -97,7 +98,7 @@ class BaseCache: :param plugins: list of :class:`aiocache.plugins.BasePlugin` derived classes. Default is empty list. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None + the backend. Default is an empty string, "". :param key_builder: alternative callable to build the key. Receives the key and the namespace as params and should return a string that can be used as a key by the underlying backend. :param timeout: int or float in seconds specifying maximum timeout for the operations to last. @@ -109,18 +110,26 @@ class BaseCache: NAME: str def __init__( - self, serializer=None, plugins=None, namespace=None, key_builder=None, timeout=5, ttl=None + self, + serializer: Optional[BaseSerializer] = None, + plugins: Optional[List[Any]] = None, # aiocache.plugins depends on aiocache.base + namespace: str = "", + key_builder: Optional[Callable[[str, str], str]] = None, + timeout: Optional[float] = 5, + ttl: Optional[float] = None, ): - self.timeout = float(timeout) if timeout is not None else timeout - self.namespace = namespace - self.ttl = float(ttl) if ttl is not None else ttl - self._build_key = key_builder or self._build_key_default + self.timeout: Optional[float] = float(timeout) if timeout is not None else None + self.ttl: Optional[float] = float(ttl) if ttl is not None else None - self._serializer = None - self.serializer = serializer or serializers.StringSerializer() + self.namespace: str = namespace + self._build_key: Callable[[str, str], str] = key_builder or ( + lambda key, namespace="": f"{namespace}{key}") - self._plugins = None - self.plugins = plugins or [] + self._serializer: Optional[BaseSerializer] = None + self.serializer: BaseSerializer = serializer or StringSerializer() + + self._plugins: List[Any] = None + self.plugins: List[Any] = plugins or [] @property def serializer(self): @@ -489,13 +498,13 @@ async def close(self, *args, _conn=None, **kwargs): async def _close(self, *args, **kwargs): pass - def build_key(self, key, namespace=None): + def build_key(self, key: str, namespace: Optional[str] = None) -> str: key_name = key.value if isinstance(key, Enum) else key ns = namespace if namespace is not None else self.namespace return self._build_key(key_name, namespace=ns) - def _build_key_default(self, key, namespace=None): - return "{}{}".format(namespace or "", key) + def _build_key_default(self, key: str, namespace: str = "") -> str: + return f"{namespace}{key}" def _get_ttl(self, ttl): return ttl if ttl is not SENTINEL else self.ttl diff --git a/aiocache/decorators.py b/aiocache/decorators.py index 11122cda..f98c1fd5 100644 --- a/aiocache/decorators.py +++ b/aiocache/decorators.py @@ -35,7 +35,7 @@ class cached: :param ttl: int seconds to store the function call. Default is None which means no expiration. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None + the backend. Default is an empty string, "". :param key_builder: Callable that allows to build the function dynamically. It receives the function plus same args and kwargs passed to the function. This behavior is necessarily different than ``BaseCache.build_key()`` @@ -176,7 +176,7 @@ class cached_stampede(cached): :param ttl: int seconds to store the function call. Default is None which means no expiration. :param key_from_attr: str arg or kwarg name from the function to use as a key. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None + the backend. Default is an empty string, "". :param key_builder: Callable that allows to build the function dynamically. It receives the function plus same args and kwargs passed to the function. This behavior is necessarily different than ``BaseCache.build_key()`` @@ -278,7 +278,7 @@ class multi_cached: :param keys_from_attr: name of the arg or kwarg in the decorated callable that contains an iterable that yields the keys returned by the decorated callable. :param namespace: string to use as default prefix for the key used in all operations of - the backend. Default is None + the backend. Default is an empty string, "". :param key_builder: Callable that enables mapping the decorated function's keys to the keys used by the cache. Receives a key from the iterable corresponding to ``keys_from_attr``, the decorated callable, and the positional and keyword arguments From 84ce34c07b1e2961583c9f3beeb2c146da2ba3d4 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 Feb 2023 08:14:15 -0800 Subject: [PATCH 13/34] mypy annotations for build_key() with namespace * BaseCache.KeyType is return type from self.build_key() * Subclasses should specify either str or bytes * BaseCache._build_key() needs no default for 'namespace' parameter * BaseCache._serializer and BaseCache.serializer now used consistently * BaseCache._plugins and BaseCache.plugins now used consistently * Clarify type annotations * Simplified tests/ut/test_base.py::TestBaseCache::... * test_alt_build_key_override_namespace() * test_alt_build_key_default_namespace() --- aiocache/backends/memcached.py | 14 ++++++++-- aiocache/backends/memory.py | 4 ++- aiocache/backends/redis.py | 21 ++++++++------ aiocache/base.py | 51 ++++++++++++++++++---------------- tests/ut/test_base.py | 23 ++++++--------- 5 files changed, 61 insertions(+), 52 deletions(-) diff --git a/aiocache/backends/memcached.py b/aiocache/backends/memcached.py index 6ea5378f..c5b12283 100644 --- a/aiocache/backends/memcached.py +++ b/aiocache/backends/memcached.py @@ -1,5 +1,5 @@ import asyncio -from typing import Optional +from typing import cast, Optional, Union import aiomcache @@ -7,6 +7,7 @@ from aiocache.serializers import JsonSerializer +# class MemcachedBackend(BaseCache[str]): class MemcachedBackend(BaseCache): def __init__(self, endpoint="127.0.0.1", port=11211, pool_size=2, **kwargs): super().__init__(**kwargs) @@ -120,6 +121,7 @@ async def _close(self, *args, _conn=None, **kwargs): await self.client.close() +# class MemcachedCache(Generic[CacheKeyType], MemcachedBackend): class MemcachedCache(MemcachedBackend): """ Memcached cache implementation with the following components as defaults: @@ -140,7 +142,9 @@ class MemcachedCache(MemcachedBackend): """ NAME = "memcached" + KeyType = Union[bytes, bytes] # Workaround: TypeAlias not in python <= 3.10 + # def __init__(self, serializer=None, *, key_type: CacheKeyType = bytes, **kwargs): def __init__(self, serializer=None, **kwargs): super().__init__(serializer=serializer or JsonSerializer(), **kwargs) @@ -148,8 +152,12 @@ def __init__(self, serializer=None, **kwargs): def parse_uri_path(cls, path): return {} - def build_key(self, key: str, namespace: Optional[str] = None) -> str: - ns_key = super().build_key(key, namespace=namespace).replace(" ", "_") + # def build_key(self, key: str, namespace: Optional[str] = None) -> CacheKeyType: + # ns_key = super().build_key(key, namespace=namespace).replace(" ", "_") + # return str.encode(ns_key) + + def build_key(self, key: str, namespace: Optional[str] = None) -> KeyType: + ns_key = cast(str, super().build_key(key, namespace=namespace)).replace(" ", "_") return str.encode(ns_key) def __repr__(self): # pragma: no cover diff --git a/aiocache/backends/memory.py b/aiocache/backends/memory.py index 2936086a..be7a9fae 100644 --- a/aiocache/backends/memory.py +++ b/aiocache/backends/memory.py @@ -1,10 +1,11 @@ import asyncio -from typing import Dict +from typing import Dict, Union from aiocache.base import BaseCache from aiocache.serializers import NullSerializer +# class SimpleMemoryBackend(BaseCache[str]): class SimpleMemoryBackend(BaseCache): """ Wrapper around dict operations to use it as a cache backend @@ -124,6 +125,7 @@ class SimpleMemoryCache(SimpleMemoryBackend): """ NAME = "memory" + KeyType = Union[str, str] # Workaround: TypeAlias not in python <= 3.10 def __init__(self, serializer=None, **kwargs): super().__init__(serializer=serializer or NullSerializer(), **kwargs) diff --git a/aiocache/backends/redis.py b/aiocache/backends/redis.py index 05539e27..74630d08 100644 --- a/aiocache/backends/redis.py +++ b/aiocache/backends/redis.py @@ -1,17 +1,20 @@ import itertools import warnings -from typing import Callable, Optional +from typing import Any, Callable, Optional, TYPE_CHECKING, Union import redis.asyncio as redis from redis.exceptions import ResponseError as IncrbyException +if TYPE_CHECKING: + from aiocache.serializers import BaseSerializer from aiocache.base import BaseCache -from aiocache.serializers import BaseSerializer, JsonSerializer +from aiocache.serializers import JsonSerializer _NOT_SET = object() +# class RedisBackend(BaseCache[str]): class RedisBackend(BaseCache): RELEASE_SCRIPT = ( "if redis.call('get',KEYS[1]) == ARGV[1] then" @@ -199,20 +202,23 @@ class RedisCache(RedisBackend): """ NAME = "redis" + KeyType = Union[str, str] # Workaround: TypeAlias not in python <= 3.10 def __init__( self, - serializer: Optional[BaseSerializer] = None, + serializer: "Optional[BaseSerializer]" = None, namespace: str = "", key_builder: Optional[Callable[[str, str], str]] = None, - **kwargs, + **kwargs: Any, ): super().__init__( serializer=serializer or JsonSerializer(), namespace=namespace, key_builder=key_builder or ( - lambda key, namespace="": f"{namespace}:{key}" if namespace else key), - **kwargs) + lambda key, namespace: f"{namespace}:{key}" if namespace else key + ), + **kwargs, + ) @classmethod def parse_uri_path(cls, path): @@ -230,8 +236,5 @@ def parse_uri_path(cls, path): options["db"] = db return options - def _build_key_default(self, key: str, namespace: str = "") -> str: - return f"{namespace}:{key}" if namespace else key - def __repr__(self): # pragma: no cover return "RedisCache ({}:{})".format(self.endpoint, self.port) diff --git a/aiocache/base.py b/aiocache/base.py index c4243350..5449ab8d 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -5,15 +5,18 @@ import time from enum import Enum from types import TracebackType -from typing import Any, Callable, List, Optional, Set, Type +from typing import Callable, List, Optional, Set, Type, TYPE_CHECKING, Union -# from aiocache.plugins import BasePlugin -from aiocache.serializers import BaseSerializer, StringSerializer +if TYPE_CHECKING: + from aiocache.plugins import BasePlugin + from aiocache.serializers import BaseSerializer +from aiocache.serializers import StringSerializer logger = logging.getLogger(__name__) SENTINEL = object() +# CacheKeyType = TypeVar("KeyType", bound=Union[str, bytes]) class API: @@ -88,6 +91,7 @@ async def _plugins(self, *args, **kwargs): return _plugins +# class BaseCache(Generic[CacheKeyType]): class BaseCache: """ Base class that agregates the common logic for the different caches that may exist. Cache @@ -108,28 +112,29 @@ class BaseCache: """ NAME: str + KeyType = Union[str, bytes] def __init__( self, - serializer: Optional[BaseSerializer] = None, - plugins: Optional[List[Any]] = None, # aiocache.plugins depends on aiocache.base + serializer: "Optional[BaseSerializer]" = None, + plugins: "Optional[List[BasePlugin]]" = None, namespace: str = "", key_builder: Optional[Callable[[str, str], str]] = None, timeout: Optional[float] = 5, ttl: Optional[float] = None, + # *, + # key_type: CacheKeyType = str, ): - self.timeout: Optional[float] = float(timeout) if timeout is not None else None - self.ttl: Optional[float] = float(ttl) if ttl is not None else None + self.timeout = float(timeout) if timeout is not None else None + self.ttl = float(ttl) if ttl is not None else None self.namespace: str = namespace self._build_key: Callable[[str, str], str] = key_builder or ( - lambda key, namespace="": f"{namespace}{key}") - - self._serializer: Optional[BaseSerializer] = None - self.serializer: BaseSerializer = serializer or StringSerializer() + lambda key, namespace: f"{namespace}{key}" + ) - self._plugins: List[Any] = None - self.plugins: List[Any] = plugins or [] + self._serializer = serializer or StringSerializer() + self._plugins = plugins or [] @property def serializer(self): @@ -171,7 +176,7 @@ async def add(self, key, value, ttl=SENTINEL, dumps_fn=None, namespace=None, _co - :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - dumps = dumps_fn or self._serializer.dumps + dumps = dumps_fn or self.serializer.dumps ns_key = self.build_key(key, namespace=namespace) await self._add(ns_key, dumps(value), ttl=self._get_ttl(ttl), _conn=_conn) @@ -200,7 +205,7 @@ async def get(self, key, default=None, loads_fn=None, namespace=None, _conn=None :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - loads = loads_fn or self._serializer.loads + loads = loads_fn or self.serializer.loads ns_key = self.build_key(key, namespace=namespace) value = loads(await self._get(ns_key, encoding=self.serializer.encoding, _conn=_conn)) @@ -231,7 +236,7 @@ async def multi_get(self, keys, loads_fn=None, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - loads = loads_fn or self._serializer.loads + loads = loads_fn or self.serializer.loads ns_keys = [self.build_key(key, namespace=namespace) for key in keys] values = [ @@ -275,7 +280,7 @@ async def set( :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - dumps = dumps_fn or self._serializer.dumps + dumps = dumps_fn or self.serializer.dumps ns_key = self.build_key(key, namespace=namespace) res = await self._set( @@ -308,7 +313,7 @@ async def multi_set(self, pairs, ttl=SENTINEL, dumps_fn=None, namespace=None, _c :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - dumps = dumps_fn or self._serializer.dumps + dumps = dumps_fn or self.serializer.dumps tmp_pairs = [] for key, value in pairs: @@ -498,13 +503,11 @@ async def close(self, *args, _conn=None, **kwargs): async def _close(self, *args, **kwargs): pass - def build_key(self, key: str, namespace: Optional[str] = None) -> str: + # def build_key(self, key: str, namespace: Optional[str] = None) -> CacheKeyType: + def build_key(self, key: str, namespace: Optional[str] = None) -> KeyType: key_name = key.value if isinstance(key, Enum) else key - ns = namespace if namespace is not None else self.namespace - return self._build_key(key_name, namespace=ns) - - def _build_key_default(self, key: str, namespace: str = "") -> str: - return f"{namespace}{key}" + ns = self.namespace if namespace is None else namespace + return self._build_key(key_name, ns) def _get_ttl(self, ttl): return ttl if ttl is not SENTINEL else self.ttl diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index bd929817..28a2156a 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -214,31 +214,25 @@ def test_alt_build_key(self): cache = BaseCache(key_builder=lambda key, namespace: "x") assert cache.build_key(Keys.KEY, "namespace") == "x" - @pytest.fixture - def alt_base_cache(self, init_namespace="test"): + def alt_build_key(self, key, namespace): """Custom key_builder for cache""" - def build_key(key, namespace=None): - sep = ":" if namespace else "" - return f'{namespace or ""}{sep}{ensure_key(key)}' - - cache = BaseCache(key_builder=build_key, namespace=init_namespace) - return cache + sep = ":" if namespace else "" + return f"{namespace}{sep}{ensure_key(key)}" @pytest.mark.parametrize( "namespace, expected", ([None, "test:" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) - def test_alt_build_key_override_namespace(self, alt_base_cache, namespace, expected): + def test_alt_build_key_override_namespace(self, namespace, expected): """Custom key_builder overrides namespace of cache""" - cache = alt_base_cache + cache = BaseCache(key_builder=self.alt_build_key, namespace="test") assert cache.build_key(Keys.KEY, namespace=namespace) == expected @pytest.mark.parametrize( - "init_namespace, expected", + "namespace, expected", ([None, ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["test", "test:" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) - async def test_alt_build_key_default_namespace( - self, init_namespace, alt_base_cache, expected): + async def test_alt_build_key_default_namespace(self, namespace, expected): """Custom key_builder for cache with or without namespace specified. Cache member functions that accept a ``namespace`` parameter @@ -250,8 +244,7 @@ async def test_alt_build_key_default_namespace( even when that cache is supplied to a lock or to a decorator using the ``alias`` argument. """ - cache = alt_base_cache - cache.namespace = init_namespace + cache = BaseCache(key_builder=self.alt_build_key, namespace=namespace) # Verify that private members are called with the correct ns_key await self._assert_add__alt_build_key_default_namespace(cache, expected) From 16db995511f393f8caa5467cc8ab81acfdf89646 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Thu, 23 Feb 2023 10:26:23 -0800 Subject: [PATCH 14/34] Define BaseCache as generic, typed by backend cache key BaseCache should be instantiated by derived classes with either 'str' or 'bytes' returned by 'build_key()'. --- aiocache/backends/memcached.py | 17 +++++------------ aiocache/backends/memory.py | 10 ++++++---- aiocache/backends/redis.py | 10 ++++++---- aiocache/base.py | 18 +++++++++--------- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/aiocache/backends/memcached.py b/aiocache/backends/memcached.py index c5b12283..7692d9fe 100644 --- a/aiocache/backends/memcached.py +++ b/aiocache/backends/memcached.py @@ -1,5 +1,5 @@ import asyncio -from typing import cast, Optional, Union +from typing import Optional import aiomcache @@ -7,8 +7,7 @@ from aiocache.serializers import JsonSerializer -# class MemcachedBackend(BaseCache[str]): -class MemcachedBackend(BaseCache): +class MemcachedBackend(BaseCache[bytes]): def __init__(self, endpoint="127.0.0.1", port=11211, pool_size=2, **kwargs): super().__init__(**kwargs) self.endpoint = endpoint @@ -121,7 +120,7 @@ async def _close(self, *args, _conn=None, **kwargs): await self.client.close() -# class MemcachedCache(Generic[CacheKeyType], MemcachedBackend): +# class MemcachedCache(MemcachedBackend): class MemcachedCache(MemcachedBackend): """ Memcached cache implementation with the following components as defaults: @@ -142,9 +141,7 @@ class MemcachedCache(MemcachedBackend): """ NAME = "memcached" - KeyType = Union[bytes, bytes] # Workaround: TypeAlias not in python <= 3.10 - # def __init__(self, serializer=None, *, key_type: CacheKeyType = bytes, **kwargs): def __init__(self, serializer=None, **kwargs): super().__init__(serializer=serializer or JsonSerializer(), **kwargs) @@ -152,12 +149,8 @@ def __init__(self, serializer=None, **kwargs): def parse_uri_path(cls, path): return {} - # def build_key(self, key: str, namespace: Optional[str] = None) -> CacheKeyType: - # ns_key = super().build_key(key, namespace=namespace).replace(" ", "_") - # return str.encode(ns_key) - - def build_key(self, key: str, namespace: Optional[str] = None) -> KeyType: - ns_key = cast(str, super().build_key(key, namespace=namespace)).replace(" ", "_") + def build_key(self, key: str, namespace: Optional[str] = None) -> bytes: + ns_key = self._str_build_key(key, namespace).replace(" ", "_") return str.encode(ns_key) def __repr__(self): # pragma: no cover diff --git a/aiocache/backends/memory.py b/aiocache/backends/memory.py index be7a9fae..a2f48579 100644 --- a/aiocache/backends/memory.py +++ b/aiocache/backends/memory.py @@ -1,18 +1,18 @@ import asyncio -from typing import Dict, Union +from typing import Dict from aiocache.base import BaseCache from aiocache.serializers import NullSerializer -# class SimpleMemoryBackend(BaseCache[str]): -class SimpleMemoryBackend(BaseCache): +class SimpleMemoryBackend(BaseCache[str]): """ Wrapper around dict operations to use it as a cache backend """ def __init__(self, **kwargs): super().__init__(**kwargs) + self.build_key = self._str_build_key self._cache: Dict[str, object] = {} self._handlers: Dict[str, asyncio.TimerHandle] = {} @@ -125,7 +125,6 @@ class SimpleMemoryCache(SimpleMemoryBackend): """ NAME = "memory" - KeyType = Union[str, str] # Workaround: TypeAlias not in python <= 3.10 def __init__(self, serializer=None, **kwargs): super().__init__(serializer=serializer or NullSerializer(), **kwargs) @@ -133,3 +132,6 @@ def __init__(self, serializer=None, **kwargs): @classmethod def parse_uri_path(cls, path): return {} + + # def build_key(self, key: str, namespace: Optional[str] = None) -> str: + # return self._str_build_key(key, namespace) diff --git a/aiocache/backends/redis.py b/aiocache/backends/redis.py index 74630d08..2a2fa0e9 100644 --- a/aiocache/backends/redis.py +++ b/aiocache/backends/redis.py @@ -1,6 +1,6 @@ import itertools import warnings -from typing import Any, Callable, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Optional, TYPE_CHECKING import redis.asyncio as redis from redis.exceptions import ResponseError as IncrbyException @@ -14,8 +14,7 @@ _NOT_SET = object() -# class RedisBackend(BaseCache[str]): -class RedisBackend(BaseCache): +class RedisBackend(BaseCache[str]): RELEASE_SCRIPT = ( "if redis.call('get',KEYS[1]) == ARGV[1] then" " return redis.call('del',KEYS[1])" @@ -48,6 +47,7 @@ def __init__( **kwargs, ): super().__init__(**kwargs) + self.build_key = self._str_build_key if pool_min_size is not _NOT_SET: warnings.warn( "Parameter 'pool_min_size' is deprecated since aiocache 0.12", @@ -202,7 +202,6 @@ class RedisCache(RedisBackend): """ NAME = "redis" - KeyType = Union[str, str] # Workaround: TypeAlias not in python <= 3.10 def __init__( self, @@ -238,3 +237,6 @@ def parse_uri_path(cls, path): def __repr__(self): # pragma: no cover return "RedisCache ({}:{})".format(self.endpoint, self.port) + + # def build_key(self, key: str, namespace: Optional[str] = None) -> str: + # return self._str_build_key(key, namespace) diff --git a/aiocache/base.py b/aiocache/base.py index 5449ab8d..a5bbc487 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -3,9 +3,10 @@ import logging import os import time +from abc import abstractmethod from enum import Enum from types import TracebackType -from typing import Callable, List, Optional, Set, Type, TYPE_CHECKING, Union +from typing import Callable, Generic, List, Optional, Set, TYPE_CHECKING, Type, TypeVar, Union if TYPE_CHECKING: from aiocache.plugins import BasePlugin @@ -16,7 +17,7 @@ logger = logging.getLogger(__name__) SENTINEL = object() -# CacheKeyType = TypeVar("KeyType", bound=Union[str, bytes]) +CacheKeyType = TypeVar("CacheKeyType", bound=Union[str, bytes]) class API: @@ -91,8 +92,7 @@ async def _plugins(self, *args, **kwargs): return _plugins -# class BaseCache(Generic[CacheKeyType]): -class BaseCache: +class BaseCache(Generic[CacheKeyType]): """ Base class that agregates the common logic for the different caches that may exist. Cache related available options are: @@ -112,7 +112,6 @@ class BaseCache: """ NAME: str - KeyType = Union[str, bytes] def __init__( self, @@ -122,8 +121,6 @@ def __init__( key_builder: Optional[Callable[[str, str], str]] = None, timeout: Optional[float] = 5, ttl: Optional[float] = None, - # *, - # key_type: CacheKeyType = str, ): self.timeout = float(timeout) if timeout is not None else None self.ttl = float(ttl) if ttl is not None else None @@ -503,8 +500,11 @@ async def close(self, *args, _conn=None, **kwargs): async def _close(self, *args, **kwargs): pass - # def build_key(self, key: str, namespace: Optional[str] = None) -> CacheKeyType: - def build_key(self, key: str, namespace: Optional[str] = None) -> KeyType: + @abstractmethod + def build_key(self, key: str, namespace: Optional[str] = None) -> CacheKeyType: + raise NotImplementedError() + + def _str_build_key(self, key: str, namespace: Optional[str] = None) -> str: key_name = key.value if isinstance(key, Enum) else key ns = self.namespace if namespace is None else namespace return self._build_key(key_name, ns) From 9dc7c77de758717eb3421cfcf78bbaeed12b351a Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Thu, 23 Feb 2023 18:11:32 -0800 Subject: [PATCH 15/34] Update tests for BaseCache with TypeVar --- tests/ut/conftest.py | 18 +++++++++++++++--- tests/ut/test_base.py | 35 ++++++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/tests/ut/conftest.py b/tests/ut/conftest.py index 0adcd1f4..19827200 100644 --- a/tests/ut/conftest.py +++ b/tests/ut/conftest.py @@ -29,14 +29,16 @@ def reset_caches(): @pytest.fixture def mock_cache(mocker): - return create_autospec(BaseCache()) + # TODO: Is there need for a separate BaseCache[bytes] fixture? + return create_autospec(BaseCache[str]()) @pytest.fixture def mock_base_cache(): """Return BaseCache instance with unimplemented methods mocked out.""" + # TODO: Is there need for a separate BaseCache[bytes] fixture? plugin = create_autospec(BasePlugin, instance=True) - cache = BaseCache(timeout=0.002, plugins=(plugin,)) + cache = BaseCache[str](timeout=0.002, plugins=(plugin,)) methods = ("_add", "_get", "_gets", "_set", "_multi_get", "_multi_set", "_delete", "_exists", "_increment", "_expire", "_clear", "_raw", "_close", "_redlock_release", "acquire_conn", "release_conn") @@ -44,12 +46,22 @@ def mock_base_cache(): for f in methods: stack.enter_context(patch.object(cache, f, autospec=True)) stack.enter_context(patch.object(cache, "_serializer", autospec=True)) + stack.enter_context(patch.object(cache, "build_key", cache._str_build_key)) yield cache +@pytest.fixture +def generic_base_cache(): + # TODO: Is there need for a separate BaseCache[bytes] fixture? + return BaseCache[str]() + + @pytest.fixture def base_cache(): - return BaseCache() + # TODO: Is there need for a separate BaseCache[bytes] fixture? + cache = BaseCache[str]() + cache.build_key = cache._str_build_key + return cache @pytest.fixture diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index 28a2156a..654eec5e 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -137,11 +137,11 @@ async def dummy(self, *args, **kwargs): class TestBaseCache: def test_str_ttl(self): - cache = BaseCache(ttl="1.5") + cache = BaseCache[str](ttl="1.5") assert cache.ttl == 1.5 def test_str_timeout(self): - cache = BaseCache(timeout="1.5") + cache = BaseCache[str](timeout="1.5") assert cache.timeout == 1.5 async def test_add(self, base_cache): @@ -197,11 +197,24 @@ async def test_acquire_conn(self, base_cache): async def test_release_conn(self, base_cache): assert await base_cache.release_conn("mock") is None + def test_build_key(self, generic_base_cache): + with pytest.raises(NotImplementedError): + generic_base_cache.build_key(Keys.KEY) + @pytest.fixture def set_test_namespace(self, base_cache): base_cache.namespace = "test" yield - base_cache.namespace = None + base_cache.namespace = "" + + @pytest.mark.parametrize( + "namespace, expected", + ([None, "None" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ) + def test_str_build_key(self, set_test_namespace, namespace, expected): + # TODO: Runtime check for namespace=None: Raise ValueError or replace with ""? + cache = BaseCache[str](namespace=namespace) + assert cache._str_build_key(Keys.KEY) == expected @pytest.mark.parametrize( "namespace, expected", @@ -210,8 +223,14 @@ def set_test_namespace(self, base_cache): def test_build_key(self, set_test_namespace, base_cache, namespace, expected): assert base_cache.build_key(Keys.KEY, namespace=namespace) == expected + def patch_str_build_key(self, cache: BaseCache[str]) -> None: + """Implement build_key() on BaseCache[str] as if it were subclassed""" + cache.build_key = cache._str_build_key + return + def test_alt_build_key(self): - cache = BaseCache(key_builder=lambda key, namespace: "x") + cache = BaseCache[str](key_builder=lambda key, namespace: "x") + self.patch_str_build_key(cache) assert cache.build_key(Keys.KEY, "namespace") == "x" def alt_build_key(self, key, namespace): @@ -225,12 +244,13 @@ def alt_build_key(self, key, namespace): ) def test_alt_build_key_override_namespace(self, namespace, expected): """Custom key_builder overrides namespace of cache""" - cache = BaseCache(key_builder=self.alt_build_key, namespace="test") + cache = BaseCache[str](key_builder=self.alt_build_key, namespace="test") + self.patch_str_build_key(cache) assert cache.build_key(Keys.KEY, namespace=namespace) == expected @pytest.mark.parametrize( "namespace, expected", - ([None, ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["test", "test:" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "None" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["test", "test:" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 ) async def test_alt_build_key_default_namespace(self, namespace, expected): """Custom key_builder for cache with or without namespace specified. @@ -244,7 +264,8 @@ async def test_alt_build_key_default_namespace(self, namespace, expected): even when that cache is supplied to a lock or to a decorator using the ``alias`` argument. """ - cache = BaseCache(key_builder=self.alt_build_key, namespace=namespace) + cache = BaseCache[str](key_builder=self.alt_build_key, namespace=namespace) + self.patch_str_build_key(cache) # Verify that private members are called with the correct ns_key await self._assert_add__alt_build_key_default_namespace(cache, expected) From a757e9a3c5c6a9d871114b16aa205e86519b5478 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Thu, 23 Feb 2023 19:02:16 -0800 Subject: [PATCH 16/34] Propagate type annotations for mypy compliance --- aiocache/__init__.py | 4 ++-- aiocache/backends/memory.py | 11 +++++++---- aiocache/backends/redis.py | 9 ++++++--- aiocache/decorators.py | 2 +- aiocache/factory.py | 4 ++-- aiocache/lock.py | 12 ++++++------ tests/ut/test_base.py | 4 ++-- 7 files changed, 26 insertions(+), 20 deletions(-) diff --git a/aiocache/__init__.py b/aiocache/__init__.py index 12503ed7..c2b5b765 100644 --- a/aiocache/__init__.py +++ b/aiocache/__init__.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, Type +from typing import Any, Dict, Type from .backends.memory import SimpleMemoryCache from .base import BaseCache @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) -AIOCACHE_CACHES: Dict[str, Type[BaseCache]] = {SimpleMemoryCache.NAME: SimpleMemoryCache} +AIOCACHE_CACHES: Dict[str, Type[BaseCache[Any]]] = {SimpleMemoryCache.NAME: SimpleMemoryCache} try: import redis diff --git a/aiocache/backends/memory.py b/aiocache/backends/memory.py index a2f48579..e13f525f 100644 --- a/aiocache/backends/memory.py +++ b/aiocache/backends/memory.py @@ -1,5 +1,5 @@ import asyncio -from typing import Dict +from typing import Dict, Optional from aiocache.base import BaseCache from aiocache.serializers import NullSerializer @@ -12,7 +12,10 @@ class SimpleMemoryBackend(BaseCache[str]): def __init__(self, **kwargs): super().__init__(**kwargs) - self.build_key = self._str_build_key + # Assigning build_key here is clear and avoids the extra stack frame + # from nesting the call to _str_build_key() in an override of build_key() + # ...but mypy needs the override definition to recognize a concrete class + # self.build_key = self._str_build_key # Simple, but not mypy-friendly self._cache: Dict[str, object] = {} self._handlers: Dict[str, asyncio.TimerHandle] = {} @@ -133,5 +136,5 @@ def __init__(self, serializer=None, **kwargs): def parse_uri_path(cls, path): return {} - # def build_key(self, key: str, namespace: Optional[str] = None) -> str: - # return self._str_build_key(key, namespace) + def build_key(self, key: str, namespace: Optional[str] = None) -> str: + return self._str_build_key(key, namespace) diff --git a/aiocache/backends/redis.py b/aiocache/backends/redis.py index 2a2fa0e9..e86f533a 100644 --- a/aiocache/backends/redis.py +++ b/aiocache/backends/redis.py @@ -47,7 +47,10 @@ def __init__( **kwargs, ): super().__init__(**kwargs) - self.build_key = self._str_build_key + # Assigning build_key here is clear and avoids the extra stack frame + # from nesting the call to _str_build_key() in an override of build_key() + # ...but mypy needs the override definition to recognize a concrete class + # self.build_key = self._str_build_key # Simple, but not mypy-friendly if pool_min_size is not _NOT_SET: warnings.warn( "Parameter 'pool_min_size' is deprecated since aiocache 0.12", @@ -238,5 +241,5 @@ def parse_uri_path(cls, path): def __repr__(self): # pragma: no cover return "RedisCache ({}:{})".format(self.endpoint, self.port) - # def build_key(self, key: str, namespace: Optional[str] = None) -> str: - # return self._str_build_key(key, namespace) + def build_key(self, key: str, namespace: Optional[str] = None) -> str: + return self._str_build_key(key, namespace) diff --git a/aiocache/decorators.py b/aiocache/decorators.py index f98c1fd5..113a70e9 100644 --- a/aiocache/decorators.py +++ b/aiocache/decorators.py @@ -144,7 +144,7 @@ def _key_from_args(self, func, args, kwargs): + str(ordered_kwargs) ) - async def get_from_cache(self, key: str): + async def get_from_cache(self, key): try: return await self.cache.get(key) except Exception: diff --git a/aiocache/factory.py b/aiocache/factory.py index 9ad2f982..0ba1a0f1 100644 --- a/aiocache/factory.py +++ b/aiocache/factory.py @@ -172,12 +172,12 @@ def get(self, alias: str) -> object: self._caches[alias] = cache return cache - def create(self, alias: str, **kwargs): + def create(self, alias, **kwargs): """Create a new cache. You can use kwargs to pass extra parameters to configure the cache. - :param alias: alias to pull configuration from + :param alias (str): alias to pull configuration from :return: New cache instance """ config = self.get_alias_config(alias) diff --git a/aiocache/lock.py b/aiocache/lock.py index d7e7e1cb..34e2299c 100644 --- a/aiocache/lock.py +++ b/aiocache/lock.py @@ -1,11 +1,11 @@ import asyncio import uuid -from typing import Any, Dict, Union +from typing import Any, Dict, Generic, Union -from aiocache.base import BaseCache +from aiocache.base import BaseCache, CacheKeyType -class RedLock: +class RedLock(Generic[CacheKeyType]): """ Implementation of `Redlock `_ with a single instance because aiocache is focused on single @@ -62,7 +62,7 @@ class RedLock: _EVENTS: Dict[str, asyncio.Event] = {} - def __init__(self, client: BaseCache, key: str, lease: Union[int, float]): + def __init__(self, client: BaseCache[CacheKeyType], key: str, lease: Union[int, float]): self.client = client self.key = self.client.build_key(key + "-lock") self.lease = lease @@ -96,7 +96,7 @@ async def _release(self): RedLock._EVENTS.pop(self.key).set() -class OptimisticLock: +class OptimisticLock(Generic[CacheKeyType]): """ Implementation of `optimistic lock `_ @@ -133,7 +133,7 @@ class OptimisticLock: If the lock is created with an unexisting key, there will never be conflicts. """ - def __init__(self, client: BaseCache, key: str): + def __init__(self, client: BaseCache[CacheKeyType], key: str): self.client = client self.key = key self.ns_key = self.client.build_key(key) diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index 654eec5e..fdc6c541 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -197,7 +197,7 @@ async def test_acquire_conn(self, base_cache): async def test_release_conn(self, base_cache): assert await base_cache.release_conn("mock") is None - def test_build_key(self, generic_base_cache): + def test_generic_build_key(self, generic_base_cache): with pytest.raises(NotImplementedError): generic_base_cache.build_key(Keys.KEY) @@ -227,7 +227,7 @@ def patch_str_build_key(self, cache: BaseCache[str]) -> None: """Implement build_key() on BaseCache[str] as if it were subclassed""" cache.build_key = cache._str_build_key return - + def test_alt_build_key(self): cache = BaseCache[str](key_builder=lambda key, namespace: "x") self.patch_str_build_key(cache) From 54093a10bc140bc072f55aafde638b5507c64af1 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Thu, 23 Feb 2023 19:19:49 -0800 Subject: [PATCH 17/34] Resolve mypy type annotations in tests --- tests/performance/test_footprint.py | 4 +++- tests/ut/backends/test_memcached.py | 2 +- tests/ut/test_base.py | 10 +++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/performance/test_footprint.py b/tests/performance/test_footprint.py index c6dabb06..4eaebf56 100644 --- a/tests/performance/test_footprint.py +++ b/tests/performance/test_footprint.py @@ -1,5 +1,6 @@ import platform import time +from typing import Any, AsyncGenerator import aiomcache import pytest @@ -7,7 +8,8 @@ @pytest.fixture -async def redis_client() -> redis.Redis: +async def redis_client() -> AsyncGenerator[redis.Redis[Any], None]: + r: redis.Redis[Any] async with redis.Redis(host="127.0.0.1", port=6379, max_connections=1) as r: yield r diff --git a/tests/ut/backends/test_memcached.py b/tests/ut/backends/test_memcached.py index 52a764b7..ba027d59 100644 --- a/tests/ut/backends/test_memcached.py +++ b/tests/ut/backends/test_memcached.py @@ -249,7 +249,7 @@ def test_parse_uri_path(self): @pytest.mark.parametrize( "namespace, expected", - ([None, "test" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "test" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_build_key_bytes(self, set_test_namespace, memcached_cache, namespace, expected): assert memcached_cache.build_key(Keys.KEY, namespace=namespace) == expected.encode() diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index fdc6c541..3bdcb671 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -209,7 +209,7 @@ def set_test_namespace(self, base_cache): @pytest.mark.parametrize( "namespace, expected", - ([None, "None" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "None" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_str_build_key(self, set_test_namespace, namespace, expected): # TODO: Runtime check for namespace=None: Raise ValueError or replace with ""? @@ -218,14 +218,14 @@ def test_str_build_key(self, set_test_namespace, namespace, expected): @pytest.mark.parametrize( "namespace, expected", - ([None, "test" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "test" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_build_key(self, set_test_namespace, base_cache, namespace, expected): assert base_cache.build_key(Keys.KEY, namespace=namespace) == expected def patch_str_build_key(self, cache: BaseCache[str]) -> None: """Implement build_key() on BaseCache[str] as if it were subclassed""" - cache.build_key = cache._str_build_key + cache.build_key = cache._str_build_key # type: ignore[assignment] return def test_alt_build_key(self): @@ -240,7 +240,7 @@ def alt_build_key(self, key, namespace): @pytest.mark.parametrize( "namespace, expected", - ([None, "test:" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "test:" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_alt_build_key_override_namespace(self, namespace, expected): """Custom key_builder overrides namespace of cache""" @@ -250,7 +250,7 @@ def test_alt_build_key_override_namespace(self, namespace, expected): @pytest.mark.parametrize( "namespace, expected", - ([None, "None" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["test", "test:" + ensure_key(Keys.KEY)]), # type: ignore[attr-defined] # noqa: B950 + ([None, "None" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["test", "test:" + ensure_key(Keys.KEY)]), # noqa: B950 ) async def test_alt_build_key_default_namespace(self, namespace, expected): """Custom key_builder for cache with or without namespace specified. From d00a18be7a5e38dfe1ca42f43a927e6e3e82641e Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Thu, 23 Feb 2023 19:23:20 -0800 Subject: [PATCH 18/34] Default namespace is empty string for decorators --- aiocache/decorators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiocache/decorators.py b/aiocache/decorators.py index 113a70e9..803a8796 100644 --- a/aiocache/decorators.py +++ b/aiocache/decorators.py @@ -60,7 +60,7 @@ class cached: def __init__( self, ttl=SENTINEL, - namespace=None, + namespace="", key_builder=None, skip_cache_func=lambda x: False, cache=Cache.MEMORY, @@ -303,7 +303,7 @@ class multi_cached: def __init__( self, keys_from_attr, - namespace=None, + namespace="", key_builder=None, skip_cache_func=lambda k, v: False, ttl=SENTINEL, From 7239f164c289998bf846d0c824bb79546fe041a9 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Thu, 23 Feb 2023 19:40:58 -0800 Subject: [PATCH 19/34] Call build_key() with namespace as positional argument --- aiocache/base.py | 18 +++++++++--------- tests/ut/backends/test_memcached.py | 2 +- tests/ut/backends/test_redis.py | 2 +- tests/ut/test_base.py | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/aiocache/base.py b/aiocache/base.py index a5bbc487..7a34c5c1 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -174,7 +174,7 @@ async def add(self, key, value, ttl=SENTINEL, dumps_fn=None, namespace=None, _co """ start = time.monotonic() dumps = dumps_fn or self.serializer.dumps - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) await self._add(ns_key, dumps(value), ttl=self._get_ttl(ttl), _conn=_conn) @@ -203,7 +203,7 @@ async def get(self, key, default=None, loads_fn=None, namespace=None, _conn=None """ start = time.monotonic() loads = loads_fn or self.serializer.loads - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) value = loads(await self._get(ns_key, encoding=self.serializer.encoding, _conn=_conn)) @@ -235,7 +235,7 @@ async def multi_get(self, keys, loads_fn=None, namespace=None, _conn=None): start = time.monotonic() loads = loads_fn or self.serializer.loads - ns_keys = [self.build_key(key, namespace=namespace) for key in keys] + ns_keys = [self.build_key(key, namespace) for key in keys] values = [ loads(value) for value in await self._multi_get( @@ -278,7 +278,7 @@ async def set( """ start = time.monotonic() dumps = dumps_fn or self.serializer.dumps - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) res = await self._set( ns_key, dumps(value), ttl=self._get_ttl(ttl), _cas_token=_cas_token, _conn=_conn @@ -314,7 +314,7 @@ async def multi_set(self, pairs, ttl=SENTINEL, dumps_fn=None, namespace=None, _c tmp_pairs = [] for key, value in pairs: - tmp_pairs.append((self.build_key(key, namespace=namespace), dumps(value))) + tmp_pairs.append((self.build_key(key, namespace), dumps(value))) await self._multi_set(tmp_pairs, ttl=self._get_ttl(ttl), _conn=_conn) @@ -345,7 +345,7 @@ async def delete(self, key, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) ret = await self._delete(ns_key, _conn=_conn) logger.debug("DELETE %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret @@ -369,7 +369,7 @@ async def exists(self, key, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) ret = await self._exists(ns_key, _conn=_conn) logger.debug("EXISTS %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret @@ -396,7 +396,7 @@ async def increment(self, key, delta=1, namespace=None, _conn=None): :raises: :class:`TypeError` if value is not incrementable """ start = time.monotonic() - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) ret = await self._increment(ns_key, delta, _conn=_conn) logger.debug("INCREMENT %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret @@ -421,7 +421,7 @@ async def expire(self, key, ttl, namespace=None, _conn=None): :raises: :class:`asyncio.TimeoutError` if it lasts more than self.timeout """ start = time.monotonic() - ns_key = self.build_key(key, namespace=namespace) + ns_key = self.build_key(key, namespace) ret = await self._expire(ns_key, ttl, _conn=_conn) logger.debug("EXPIRE %s %d (%.4f)s", ns_key, ret, time.monotonic() - start) return ret diff --git a/tests/ut/backends/test_memcached.py b/tests/ut/backends/test_memcached.py index ba027d59..81cd744c 100644 --- a/tests/ut/backends/test_memcached.py +++ b/tests/ut/backends/test_memcached.py @@ -252,7 +252,7 @@ def test_parse_uri_path(self): ([None, "test" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_build_key_bytes(self, set_test_namespace, memcached_cache, namespace, expected): - assert memcached_cache.build_key(Keys.KEY, namespace=namespace) == expected.encode() + assert memcached_cache.build_key(Keys.KEY, namespace) == expected.encode() def test_build_key_no_namespace(self, memcached_cache): assert memcached_cache.build_key(Keys.KEY, namespace=None) == Keys.KEY.encode() diff --git a/tests/ut/backends/test_redis.py b/tests/ut/backends/test_redis.py index 8f2b6337..c6ad755a 100644 --- a/tests/ut/backends/test_redis.py +++ b/tests/ut/backends/test_redis.py @@ -256,7 +256,7 @@ def test_parse_uri_path(self, path, expected): ([None, "test:" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns:" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_build_key_double_dot(self, set_test_namespace, redis_cache, namespace, expected): - assert redis_cache.build_key(Keys.KEY, namespace=namespace) == expected + assert redis_cache.build_key(Keys.KEY, namespace) == expected def test_build_key_no_namespace(self, redis_cache): assert redis_cache.build_key(Keys.KEY, namespace=None) == Keys.KEY diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index 3bdcb671..8e890dee 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -221,7 +221,7 @@ def test_str_build_key(self, set_test_namespace, namespace, expected): ([None, "test" + ensure_key(Keys.KEY)], ["", ensure_key(Keys.KEY)], ["my_ns", "my_ns" + ensure_key(Keys.KEY)]), # noqa: B950 ) def test_build_key(self, set_test_namespace, base_cache, namespace, expected): - assert base_cache.build_key(Keys.KEY, namespace=namespace) == expected + assert base_cache.build_key(Keys.KEY, namespace) == expected def patch_str_build_key(self, cache: BaseCache[str]) -> None: """Implement build_key() on BaseCache[str] as if it were subclassed""" @@ -246,7 +246,7 @@ def test_alt_build_key_override_namespace(self, namespace, expected): """Custom key_builder overrides namespace of cache""" cache = BaseCache[str](key_builder=self.alt_build_key, namespace="test") self.patch_str_build_key(cache) - assert cache.build_key(Keys.KEY, namespace=namespace) == expected + assert cache.build_key(Keys.KEY, namespace) == expected @pytest.mark.parametrize( "namespace, expected", From 5995d64ffd4c8c7680c46949fc9fad2a511a3eca Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Thu, 23 Feb 2023 19:44:11 -0800 Subject: [PATCH 20/34] redis.asyncio.Redis is generic for type checking purposes, but not at runtime --- tests/performance/test_footprint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/test_footprint.py b/tests/performance/test_footprint.py index 4eaebf56..8777d6bc 100644 --- a/tests/performance/test_footprint.py +++ b/tests/performance/test_footprint.py @@ -8,8 +8,8 @@ @pytest.fixture -async def redis_client() -> AsyncGenerator[redis.Redis[Any], None]: - r: redis.Redis[Any] +async def redis_client() -> "AsyncGenerator[redis.Redis[Any], None]": + r: "redis.Redis[Any]" async with redis.Redis(host="127.0.0.1", port=6379, max_connections=1) as r: yield r From e8c23068b44197479c651d7a48e75d8f16dee53b Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Thu, 23 Feb 2023 20:05:20 -0800 Subject: [PATCH 21/34] Update alt_key_builder example with revised build_key() logickey_builder --- examples/alt_key_builder.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/alt_key_builder.py b/examples/alt_key_builder.py index 81dc09ad..aeb984d9 100644 --- a/examples/alt_key_builder.py +++ b/examples/alt_key_builder.py @@ -21,13 +21,14 @@ Args: key (str): undecorated key name - namespace (str, optional): Prefix to add to the key. Defaults to None. + namespace (str, optional): Prefix to add to the key. Defaults to "". Returns: + (str or bytes) By default, ``cache.build_key()`` returns ``f'{namespace}{sep}{key}'``, where some backends might include an optional separator, ``sep``. Some backends might strip or replace illegal characters, and encode - the result before returning it. Typically str or bytes. + the result before returning it. -------------------------------------------------------------------------- 2. Custom ``key_builder`` for a cache decorator automatically generates a @@ -59,21 +60,21 @@ async def demo_key_builders(): # 1. Custom ``key_builder`` for a cache # ------------------------------------- -def ensure_no_spaces(key, namespace=None, replace='_'): +def ensure_no_spaces(key, namespace, replace='_'): """Prefix key with namespace; replace each space with ``replace``""" - aggregate_key = f"{namespace or ''}{key}" + aggregate_key = f"{namespace}{key}" custom_key = aggregate_key.replace(' ', replace) return custom_key -def bytes_key(key, namespace=None): +def bytes_key(key, namespace): """Prefix key with namespace; convert output to bytes""" - aggregate_key = f"{namespace or ''}{key}" + aggregate_key = f"{namespace}{key}" custom_key = aggregate_key.encode() return custom_key -def fixed_key(key, namespace=None): +def fixed_key(key, namespace): """Ignore input, generate a fixed key""" unchanging_key = "universal key" return unchanging_key From 4126d60cfa05b29ea4a09a4ca771c6ccfefece56 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Fri, 24 Feb 2023 06:02:40 -0800 Subject: [PATCH 22/34] Remove unneeded typing in decorator tests [mypy] --- tests/ut/test_decorators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ut/test_decorators.py b/tests/ut/test_decorators.py index 4b0ebdab..bf48a2b0 100644 --- a/tests/ut/test_decorators.py +++ b/tests/ut/test_decorators.py @@ -60,7 +60,7 @@ def test_fails_at_instantiation(self): with pytest.raises(TypeError): @cached(wrong_param=1) - async def fn() -> None: + async def fn(): """Dummy function.""" def test_alias_takes_precedence(self, mock_cache): @@ -373,7 +373,7 @@ def test_fails_at_instantiation(self): with pytest.raises(TypeError): @multi_cached(wrong_param=1) - async def fn() -> None: + async def fn(): """Dummy function.""" def test_alias_takes_precedence(self, mock_cache): From 3563defbbd3aa521e447a672be971ca01c134b1e Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Fri, 24 Feb 2023 10:03:41 -0800 Subject: [PATCH 23/34] BaseCache key_builder param defaults to lambda --- aiocache/base.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/aiocache/base.py b/aiocache/base.py index 7a34c5c1..0dab5df6 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -118,7 +118,7 @@ def __init__( serializer: "Optional[BaseSerializer]" = None, plugins: "Optional[List[BasePlugin]]" = None, namespace: str = "", - key_builder: Optional[Callable[[str, str], str]] = None, + key_builder: Callable[[str, str], str] = lambda key, namespace: f"{namespace}{key}", timeout: Optional[float] = 5, ttl: Optional[float] = None, ): @@ -126,9 +126,7 @@ def __init__( self.ttl = float(ttl) if ttl is not None else None self.namespace: str = namespace - self._build_key: Callable[[str, str], str] = key_builder or ( - lambda key, namespace: f"{namespace}{key}" - ) + self._build_key = key_builder self._serializer = serializer or StringSerializer() self._plugins = plugins or [] From 07e1919e6946c573f04a7d9c2382e09360ff6186 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Fri, 24 Feb 2023 10:07:48 -0800 Subject: [PATCH 24/34] Remove extraneous comments --- aiocache/backends/memcached.py | 1 - aiocache/backends/memory.py | 4 ---- aiocache/backends/redis.py | 4 ---- 3 files changed, 9 deletions(-) diff --git a/aiocache/backends/memcached.py b/aiocache/backends/memcached.py index 7692d9fe..45e1971a 100644 --- a/aiocache/backends/memcached.py +++ b/aiocache/backends/memcached.py @@ -120,7 +120,6 @@ async def _close(self, *args, _conn=None, **kwargs): await self.client.close() -# class MemcachedCache(MemcachedBackend): class MemcachedCache(MemcachedBackend): """ Memcached cache implementation with the following components as defaults: diff --git a/aiocache/backends/memory.py b/aiocache/backends/memory.py index e13f525f..eafef29e 100644 --- a/aiocache/backends/memory.py +++ b/aiocache/backends/memory.py @@ -12,10 +12,6 @@ class SimpleMemoryBackend(BaseCache[str]): def __init__(self, **kwargs): super().__init__(**kwargs) - # Assigning build_key here is clear and avoids the extra stack frame - # from nesting the call to _str_build_key() in an override of build_key() - # ...but mypy needs the override definition to recognize a concrete class - # self.build_key = self._str_build_key # Simple, but not mypy-friendly self._cache: Dict[str, object] = {} self._handlers: Dict[str, asyncio.TimerHandle] = {} diff --git a/aiocache/backends/redis.py b/aiocache/backends/redis.py index e86f533a..f883c83c 100644 --- a/aiocache/backends/redis.py +++ b/aiocache/backends/redis.py @@ -47,10 +47,6 @@ def __init__( **kwargs, ): super().__init__(**kwargs) - # Assigning build_key here is clear and avoids the extra stack frame - # from nesting the call to _str_build_key() in an override of build_key() - # ...but mypy needs the override definition to recognize a concrete class - # self.build_key = self._str_build_key # Simple, but not mypy-friendly if pool_min_size is not _NOT_SET: warnings.warn( "Parameter 'pool_min_size' is deprecated since aiocache 0.12", From 589d2b8168a7c2cd81a1ffa1af104479618f3d6c Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Fri, 24 Feb 2023 10:26:48 -0800 Subject: [PATCH 25/34] Remove bound on CacheKeyType --- aiocache/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiocache/base.py b/aiocache/base.py index 0dab5df6..98baee02 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -6,7 +6,7 @@ from abc import abstractmethod from enum import Enum from types import TracebackType -from typing import Callable, Generic, List, Optional, Set, TYPE_CHECKING, Type, TypeVar, Union +from typing import Callable, Generic, List, Optional, Set, TYPE_CHECKING, Type, TypeVar if TYPE_CHECKING: from aiocache.plugins import BasePlugin @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) SENTINEL = object() -CacheKeyType = TypeVar("CacheKeyType", bound=Union[str, bytes]) +CacheKeyType = TypeVar("CacheKeyType") class API: From 08d8bf960e6575517855cbbbd1a5563ddd093495 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Fri, 24 Feb 2023 10:27:53 -0800 Subject: [PATCH 26/34] Correct the return annotation for redis_client fixture --- tests/performance/test_footprint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/test_footprint.py b/tests/performance/test_footprint.py index 8777d6bc..e501f4de 100644 --- a/tests/performance/test_footprint.py +++ b/tests/performance/test_footprint.py @@ -1,6 +1,6 @@ import platform import time -from typing import Any, AsyncGenerator +from typing import Any, AsyncIterator import aiomcache import pytest @@ -8,7 +8,7 @@ @pytest.fixture -async def redis_client() -> "AsyncGenerator[redis.Redis[Any], None]": +async def redis_client() -> AsyncIterator["redis.Redis[Any]"]: r: "redis.Redis[Any]" async with redis.Redis(host="127.0.0.1", port=6379, max_connections=1) as r: yield r From 08b70d7aeb3fa497ac75e66f445e57cc45e6d1ef Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Fri, 24 Feb 2023 10:31:00 -0800 Subject: [PATCH 27/34] Test for abstract BaseCache --- tests/ut/conftest.py | 3 +-- tests/ut/test_base.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/ut/conftest.py b/tests/ut/conftest.py index 19827200..053247b0 100644 --- a/tests/ut/conftest.py +++ b/tests/ut/conftest.py @@ -36,7 +36,6 @@ def mock_cache(mocker): @pytest.fixture def mock_base_cache(): """Return BaseCache instance with unimplemented methods mocked out.""" - # TODO: Is there need for a separate BaseCache[bytes] fixture? plugin = create_autospec(BasePlugin, instance=True) cache = BaseCache[str](timeout=0.002, plugins=(plugin,)) methods = ("_add", "_get", "_gets", "_set", "_multi_get", "_multi_set", "_delete", @@ -51,7 +50,7 @@ def mock_base_cache(): @pytest.fixture -def generic_base_cache(): +def abstract_base_cache(): # TODO: Is there need for a separate BaseCache[bytes] fixture? return BaseCache[str]() diff --git a/tests/ut/test_base.py b/tests/ut/test_base.py index 8e890dee..3449dabf 100644 --- a/tests/ut/test_base.py +++ b/tests/ut/test_base.py @@ -197,9 +197,9 @@ async def test_acquire_conn(self, base_cache): async def test_release_conn(self, base_cache): assert await base_cache.release_conn("mock") is None - def test_generic_build_key(self, generic_base_cache): + def test_abstract_build_key(self, abstract_base_cache): with pytest.raises(NotImplementedError): - generic_base_cache.build_key(Keys.KEY) + abstract_base_cache.build_key(Keys.KEY) @pytest.fixture def set_test_namespace(self, base_cache): From 64436a0e56fb3da534cb8a85b209eda8bdbe4fb6 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Fri, 24 Feb 2023 11:26:40 -0800 Subject: [PATCH 28/34] Revert extraneous mypy typing fixes These will be dealt with in a separate PR. --- aiocache/factory.py | 4 ++-- examples/alt_key_builder.py | 3 +-- tests/performance/test_footprint.py | 4 +--- tests/ut/conftest.py | 1 - tests/ut/test_decorators.py | 4 ++-- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/aiocache/factory.py b/aiocache/factory.py index 0ba1a0f1..9ad2f982 100644 --- a/aiocache/factory.py +++ b/aiocache/factory.py @@ -172,12 +172,12 @@ def get(self, alias: str) -> object: self._caches[alias] = cache return cache - def create(self, alias, **kwargs): + def create(self, alias: str, **kwargs): """Create a new cache. You can use kwargs to pass extra parameters to configure the cache. - :param alias (str): alias to pull configuration from + :param alias: alias to pull configuration from :return: New cache instance """ config = self.get_alias_config(alias) diff --git a/examples/alt_key_builder.py b/examples/alt_key_builder.py index aeb984d9..c0163d51 100644 --- a/examples/alt_key_builder.py +++ b/examples/alt_key_builder.py @@ -24,11 +24,10 @@ namespace (str, optional): Prefix to add to the key. Defaults to "". Returns: - (str or bytes) By default, ``cache.build_key()`` returns ``f'{namespace}{sep}{key}'``, where some backends might include an optional separator, ``sep``. Some backends might strip or replace illegal characters, and encode - the result before returning it. + the result before returning it. Typically str or bytes. -------------------------------------------------------------------------- 2. Custom ``key_builder`` for a cache decorator automatically generates a diff --git a/tests/performance/test_footprint.py b/tests/performance/test_footprint.py index e501f4de..c6dabb06 100644 --- a/tests/performance/test_footprint.py +++ b/tests/performance/test_footprint.py @@ -1,6 +1,5 @@ import platform import time -from typing import Any, AsyncIterator import aiomcache import pytest @@ -8,8 +7,7 @@ @pytest.fixture -async def redis_client() -> AsyncIterator["redis.Redis[Any]"]: - r: "redis.Redis[Any]" +async def redis_client() -> redis.Redis: async with redis.Redis(host="127.0.0.1", port=6379, max_connections=1) as r: yield r diff --git a/tests/ut/conftest.py b/tests/ut/conftest.py index 053247b0..385294c9 100644 --- a/tests/ut/conftest.py +++ b/tests/ut/conftest.py @@ -29,7 +29,6 @@ def reset_caches(): @pytest.fixture def mock_cache(mocker): - # TODO: Is there need for a separate BaseCache[bytes] fixture? return create_autospec(BaseCache[str]()) diff --git a/tests/ut/test_decorators.py b/tests/ut/test_decorators.py index bf48a2b0..4b0ebdab 100644 --- a/tests/ut/test_decorators.py +++ b/tests/ut/test_decorators.py @@ -60,7 +60,7 @@ def test_fails_at_instantiation(self): with pytest.raises(TypeError): @cached(wrong_param=1) - async def fn(): + async def fn() -> None: """Dummy function.""" def test_alias_takes_precedence(self, mock_cache): @@ -373,7 +373,7 @@ def test_fails_at_instantiation(self): with pytest.raises(TypeError): @multi_cached(wrong_param=1) - async def fn(): + async def fn() -> None: """Dummy function.""" def test_alias_takes_precedence(self, mock_cache): From bc385f79c2f6664e42a3a20c9b49dd09e82179b8 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Fri, 24 Feb 2023 11:42:10 -0800 Subject: [PATCH 29/34] Revise the changelog for PR #670 --- CHANGES.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8907f575..3df5b358 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -15,10 +15,14 @@ There are a number of backwards-incompatible changes. These points should help w * The ``key`` parameter has been removed from the ``cached`` decorator. The behaviour can be easily reimplemented with ``key_builder=lambda *a, **kw: "foo"`` * When using the ``key_builder`` parameter in ``@multicached``, the function will now return the original, unmodified keys, only using the transformed keys in the cache (this has always been the documented behaviour, but not the implemented behaviour). * ``BaseSerializer`` is now an ``ABC``, so cannot be instantiated directly. -* The logic for encoding a cache key and selecting the key's namespace is now encapsulated in -the ``build_key(key, namespace)`` member of BaseCache (and its backend subclasses). Now -creating a cache with a custom ``key_builder`` argument simply requires that function to -return any string mapping from the ``key`` and optional ``namespace`` parameters. +* Each backend cache implementation must specify the data type of its cache key, +``CachKeyType``, when deriving from ``BaseCache[CachKeyType]``. The abstract method, +``build_key()`` must be defined and return a cache key of the corresponding type. +* The logic for encoding a cache key and selecting the key's namespace is now expected to be +encapsulated in the ``build_key(key, namespace)`` member of backend cache implementations +that are derived from ``BaseCache[CachKeyType]``. Now instantiating a cache with a custom +``key_builder`` argument simply requires that function to return any string that maps from the +``key`` and ``namespace`` parameters (which are also strings). 0.12.0 (2023-01-13) From 396952f4d64398225402a61d5f6e2a058df98b75 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 27 Feb 2023 17:33:03 +0000 Subject: [PATCH 30/34] Update redis.py --- aiocache/backends/redis.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/aiocache/backends/redis.py b/aiocache/backends/redis.py index f883c83c..408989a1 100644 --- a/aiocache/backends/redis.py +++ b/aiocache/backends/redis.py @@ -5,11 +5,12 @@ import redis.asyncio as redis from redis.exceptions import ResponseError as IncrbyException -if TYPE_CHECKING: - from aiocache.serializers import BaseSerializer from aiocache.base import BaseCache from aiocache.serializers import JsonSerializer +if TYPE_CHECKING: # pragma: no cover + from aiocache.serializers import BaseSerializer + _NOT_SET = object() @@ -204,7 +205,7 @@ class RedisCache(RedisBackend): def __init__( self, - serializer: "Optional[BaseSerializer]" = None, + serializer: Optional["BaseSerializer"] = None, namespace: str = "", key_builder: Optional[Callable[[str, str], str]] = None, **kwargs: Any, From 6c4780e38b123ab2857fa2855b81e2e955212742 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 27 Feb 2023 17:38:03 +0000 Subject: [PATCH 31/34] Update base.py --- aiocache/base.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/aiocache/base.py b/aiocache/base.py index 98baee02..ce44dfbe 100644 --- a/aiocache/base.py +++ b/aiocache/base.py @@ -8,10 +8,11 @@ from types import TracebackType from typing import Callable, Generic, List, Optional, Set, TYPE_CHECKING, Type, TypeVar -if TYPE_CHECKING: +from aiocache.serializers import StringSerializer + +if TYPE_CHECKING: # pragma: no cover from aiocache.plugins import BasePlugin from aiocache.serializers import BaseSerializer -from aiocache.serializers import StringSerializer logger = logging.getLogger(__name__) @@ -115,8 +116,8 @@ class BaseCache(Generic[CacheKeyType]): def __init__( self, - serializer: "Optional[BaseSerializer]" = None, - plugins: "Optional[List[BasePlugin]]" = None, + serializer: Optional["BaseSerializer"] = None, + plugins: Optional[List["BasePlugin"]] = None, namespace: str = "", key_builder: Callable[[str, str], str] = lambda key, namespace: f"{namespace}{key}", timeout: Optional[float] = 5, @@ -125,7 +126,7 @@ def __init__( self.timeout = float(timeout) if timeout is not None else None self.ttl = float(ttl) if ttl is not None else None - self.namespace: str = namespace + self.namespace = namespace self._build_key = key_builder self._serializer = serializer or StringSerializer() @@ -503,6 +504,7 @@ def build_key(self, key: str, namespace: Optional[str] = None) -> CacheKeyType: raise NotImplementedError() def _str_build_key(self, key: str, namespace: Optional[str] = None) -> str: + """Simple key builder that can be used in subclasses for build_key().""" key_name = key.value if isinstance(key, Enum) else key ns = self.namespace if namespace is None else namespace return self._build_key(key_name, ns) From ce2f26ad1aee807071c4919a72a81973c76ebf68 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 27 Feb 2023 17:39:16 +0000 Subject: [PATCH 32/34] Update alt_key_builder.py --- examples/alt_key_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/alt_key_builder.py b/examples/alt_key_builder.py index c0163d51..de13a9ea 100644 --- a/examples/alt_key_builder.py +++ b/examples/alt_key_builder.py @@ -59,7 +59,7 @@ async def demo_key_builders(): # 1. Custom ``key_builder`` for a cache # ------------------------------------- -def ensure_no_spaces(key, namespace, replace='_'): +def ensure_no_spaces(key, namespace, replace="_"): """Prefix key with namespace; replace each space with ``replace``""" aggregate_key = f"{namespace}{key}" custom_key = aggregate_key.replace(' ', replace) From 3d506116b8818b5eabd9c84a43190e0b2d2b8599 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 27 Feb 2023 18:09:02 +0000 Subject: [PATCH 33/34] Update CHANGES.rst --- CHANGES.rst | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3df5b358..0feacfed 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,17 +12,13 @@ Migration instructions There are a number of backwards-incompatible changes. These points should help with migrating from an older release: +* The ``key_builder`` parameter now expects a callback which accepts 2 strings and returns a string in all cache implementations, making the builders simpler and interchangeable. * The ``key`` parameter has been removed from the ``cached`` decorator. The behaviour can be easily reimplemented with ``key_builder=lambda *a, **kw: "foo"`` * When using the ``key_builder`` parameter in ``@multicached``, the function will now return the original, unmodified keys, only using the transformed keys in the cache (this has always been the documented behaviour, but not the implemented behaviour). * ``BaseSerializer`` is now an ``ABC``, so cannot be instantiated directly. -* Each backend cache implementation must specify the data type of its cache key, -``CachKeyType``, when deriving from ``BaseCache[CachKeyType]``. The abstract method, -``build_key()`` must be defined and return a cache key of the corresponding type. -* The logic for encoding a cache key and selecting the key's namespace is now expected to be -encapsulated in the ``build_key(key, namespace)`` member of backend cache implementations -that are derived from ``BaseCache[CachKeyType]``. Now instantiating a cache with a custom -``key_builder`` argument simply requires that function to return any string that maps from the -``key`` and ``namespace`` parameters (which are also strings). +* If subclassing ``BaseCache`` to implement a custom backend: + * The cache key type used by the backend must now be specified when inheriting (e.g. ``BaseCache[str]`` typically). + * The ``build_key()`` method must now be defined (this should generally involve calling ``self._str_build_key()`` as a helper). 0.12.0 (2023-01-13) From 23cd481b7eb29391a6c34c15ee82737123d1b5b9 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 27 Feb 2023 18:10:25 +0000 Subject: [PATCH 34/34] Update CHANGES.rst --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 0feacfed..bc52e18a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,7 @@ There are a number of backwards-incompatible changes. These points should help w * When using the ``key_builder`` parameter in ``@multicached``, the function will now return the original, unmodified keys, only using the transformed keys in the cache (this has always been the documented behaviour, but not the implemented behaviour). * ``BaseSerializer`` is now an ``ABC``, so cannot be instantiated directly. * If subclassing ``BaseCache`` to implement a custom backend: + * The cache key type used by the backend must now be specified when inheriting (e.g. ``BaseCache[str]`` typically). * The ``build_key()`` method must now be defined (this should generally involve calling ``self._str_build_key()`` as a helper).