diff --git a/docs/source/api.rst b/docs/source/api.rst index 737085c21a..c8d7dd7abb 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -18,6 +18,11 @@ sopel.tools.identifiers .. automodule:: sopel.tools.identifiers :members: +sopel.tools.memories +-------------------- + +.. automodule:: sopel.tools.memories + :members: sopel.tools.web --------------- diff --git a/sopel/bot.py b/sopel/bot.py index 716b095d9d..b69ecc8fd0 100644 --- a/sopel/bot.py +++ b/sopel/bot.py @@ -109,7 +109,7 @@ def __init__(self, config, daemon=False): self.memory = tools.SopelMemory() """ A thread-safe dict for storage of runtime data to be shared between - plugins. See :class:`sopel.tools.SopelMemory`. + plugins. See :class:`sopel.tools.memories.SopelMemory`. """ self.shutdown_methods = [] diff --git a/sopel/tools/__init__.py b/sopel/tools/__init__.py index d55234a4e7..10ff776964 100644 --- a/sopel/tools/__init__.py +++ b/sopel/tools/__init__.py @@ -14,21 +14,23 @@ from __future__ import annotations import codecs -from collections import defaultdict import logging import os import re import sys -import threading -from typing import Callable from sopel.lifecycle import deprecated # Don't delete; maintains backward compatibility with pre-8.0 API from ._events import events # NOQA -from .identifiers import Identifier -from . import time, web # NOQA +# shortcuts & backward compatibility with pre-8.0 +from .identifiers import Identifier # NOQA +from .memories import ( # NOQA + SopelIdentifierMemory, + SopelMemory, + SopelMemoryWithDefault, +) +from . import time, web # NOQA -IdentifierFactory = Callable[[str], Identifier] # Can be implementation-dependent _regex_type = type(re.compile('')) @@ -237,151 +239,6 @@ def get_logger(plugin_name): return logging.getLogger('sopel.externals.%s' % plugin_name) -class SopelMemory(dict): - """A simple thread-safe ``dict`` implementation. - - In order to prevent exceptions when iterating over the values and changing - them at the same time from different threads, we use a blocking lock in - ``__setitem__`` and ``__contains__``. - - .. versionadded:: 3.1 - As ``Willie.WillieMemory`` - .. versionchanged:: 4.0 - Moved to ``tools.WillieMemory`` - .. versionchanged:: 6.0 - Renamed from ``WillieMemory`` to ``SopelMemory`` - """ - def __init__(self, *args): - dict.__init__(self, *args) - self.lock = threading.Lock() - - def __setitem__(self, key, value): - """Set a key equal to a value. - - The dict is locked for other writes while doing so. - """ - self.lock.acquire() - result = dict.__setitem__(self, key, value) - self.lock.release() - return result - - def __contains__(self, key): - """Check if a key is in the dict. - - The dict is locked for writes while doing so. - """ - self.lock.acquire() - result = dict.__contains__(self, key) - self.lock.release() - return result - - # Needed to make it explicit that we don't care about the `lock` attribute - # when comparing/hashing SopelMemory objects. - __eq__ = dict.__eq__ - __ne__ = dict.__ne__ - __hash__ = dict.__hash__ - - -class SopelMemoryWithDefault(defaultdict): - """Same as SopelMemory, but subclasses from collections.defaultdict. - - .. versionadded:: 4.3 - As ``WillieMemoryWithDefault`` - .. versionchanged:: 6.0 - Renamed to ``SopelMemoryWithDefault`` - """ - def __init__(self, *args): - defaultdict.__init__(self, *args) - self.lock = threading.Lock() - - def __setitem__(self, key, value): - """Set a key equal to a value. - - The dict is locked for other writes while doing so. - """ - self.lock.acquire() - result = defaultdict.__setitem__(self, key, value) - self.lock.release() - return result - - def __contains__(self, key): - """Check if a key is in the dict. - - The dict is locked for writes while doing so. - """ - self.lock.acquire() - result = defaultdict.__contains__(self, key) - self.lock.release() - return result - - -class SopelIdentifierMemory(SopelMemory): - """Special Sopel memory that stores ``Identifier`` as key. - - This is a convenient subclass of :class:`SopelMemory` that always casts its - keys as instances of :class:`~.identifiers.Identifier`:: - - >>> from sopel import tools - >>> memory = tools.SopelIdentifierMemory() - >>> memory['Exirel'] = 'king' - >>> list(memory.items()) - [(Identifier('Exirel'), 'king')] - >>> tools.Identifier('exirel') in memory - True - >>> 'exirel' in memory - True - - As seen in the example above, it is possible to perform various operations - with both ``Identifier`` and :class:`str` objects, taking advantage of the - case-insensitive behavior of ``Identifier``. - - As it works with :class:`~.identifiers.Identifier`, it accepts an - identifier factory. This factory usually comes from a bot instance (see - :meth:`bot.make_identifier()`), like - in the example of a plugin setup function:: - - def setup(bot): - bot.memory['my_plugin_storage'] = SopelIdentifierMemory( - identifier_factory=bot.make_identifier, - ) - - .. note:: - - Internally, it will try to do ``key = self.make_identifier(key)``, - which will raise an exception if it cannot instantiate the key - properly:: - - >>> memory[1] = 'error' - AttributeError: 'int' object has no attribute 'translate' - - .. versionadded:: 7.1 - - .. versionchanged:: 8.0 - - The parameter ``identifier_factory`` has been added to properly - transform ``str`` into :class:`~.identifiers.Identifier`. This factory - is stored and accessible through :attr:`make_identifier`. - - """ - def __init__( - self, - *args, - identifier_factory: IdentifierFactory = Identifier, - ) -> None: - super().__init__(*args) - self.make_identifier = identifier_factory - """A factory to transform keys into identifiers.""" - - def __getitem__(self, key): - return super().__getitem__(self.make_identifier(key)) - - def __contains__(self, key): - return super().__contains__(self.make_identifier(key)) - - def __setitem__(self, key, value): - super().__setitem__(self.make_identifier(key), value) - - def chain_loaders(*lazy_loaders): """Chain lazy loaders into one. diff --git a/sopel/tools/memories.py b/sopel/tools/memories.py new file mode 100644 index 0000000000..2ee9396183 --- /dev/null +++ b/sopel/tools/memories.py @@ -0,0 +1,167 @@ +"""Thread-safe memory data-structures for Sopel. + +Sopel uses lots of threads to manage rules and jobs and other features, and it +needs to store shared information safely. This class contains various memory +classes that are thread-safe, with some convenience features. +""" +from __future__ import annotations + +from collections import defaultdict +import threading +from typing import Callable + +from .identifiers import Identifier + + +IdentifierFactory = Callable[[str], Identifier] + + +class SopelMemory(dict): + """A simple thread-safe ``dict`` implementation. + + In order to prevent exceptions when iterating over the values and changing + them at the same time from different threads, we use a blocking lock in + ``__setitem__`` and ``__contains__``. + + .. versionadded:: 3.1 + As ``Willie.WillieMemory`` + .. versionchanged:: 4.0 + Moved to ``tools.WillieMemory`` + .. versionchanged:: 6.0 + Renamed from ``WillieMemory`` to ``SopelMemory`` + .. versionchanged:: 8.0 + Moved from ``tools`` to ``tools.memories`` + """ + def __init__(self, *args): + dict.__init__(self, *args) + self.lock = threading.Lock() + + def __setitem__(self, key, value): + """Set a key equal to a value. + + The dict is locked for other writes while doing so. + """ + self.lock.acquire() + result = dict.__setitem__(self, key, value) + self.lock.release() + return result + + def __contains__(self, key): + """Check if a key is in the dict. + + The dict is locked for writes while doing so. + """ + self.lock.acquire() + result = dict.__contains__(self, key) + self.lock.release() + return result + + # Needed to make it explicit that we don't care about the `lock` attribute + # when comparing/hashing SopelMemory objects. + __eq__ = dict.__eq__ + __ne__ = dict.__ne__ + __hash__ = dict.__hash__ + + +class SopelMemoryWithDefault(defaultdict): + """Same as SopelMemory, but subclasses from collections.defaultdict. + + .. versionadded:: 4.3 + As ``WillieMemoryWithDefault`` + .. versionchanged:: 6.0 + Renamed to ``SopelMemoryWithDefault`` + .. versionchanged:: 8.0 + Moved from ``tools`` to ``tools.memories`` + """ + def __init__(self, *args): + defaultdict.__init__(self, *args) + self.lock = threading.Lock() + + def __setitem__(self, key, value): + """Set a key equal to a value. + + The dict is locked for other writes while doing so. + """ + self.lock.acquire() + result = defaultdict.__setitem__(self, key, value) + self.lock.release() + return result + + def __contains__(self, key): + """Check if a key is in the dict. + + The dict is locked for writes while doing so. + """ + self.lock.acquire() + result = defaultdict.__contains__(self, key) + self.lock.release() + return result + + +class SopelIdentifierMemory(SopelMemory): + """Special Sopel memory that stores ``Identifier`` as key. + + This is a convenient subclass of :class:`SopelMemory` that always casts its + keys as instances of :class:`~.identifiers.Identifier`:: + + >>> from sopel import tools + >>> memory = tools.SopelIdentifierMemory() + >>> memory['Exirel'] = 'king' + >>> list(memory.items()) + [(Identifier('Exirel'), 'king')] + >>> tools.Identifier('exirel') in memory + True + >>> 'exirel' in memory + True + + As seen in the example above, it is possible to perform various operations + with both ``Identifier`` and :class:`str` objects, taking advantage of the + case-insensitive behavior of ``Identifier``. + + As it works with :class:`~.identifiers.Identifier`, it accepts an + identifier factory. This factory usually comes from a + :class:`bot instance`, like in the example of a plugin + setup function:: + + def setup(bot): + bot.memory['my_plugin_storage'] = SopelIdentifierMemory( + identifier_factory=bot.make_identifier, + ) + + .. note:: + + Internally, it will try to do ``key = self.make_identifier(key)``, + which will raise an exception if it cannot instantiate the key + properly:: + + >>> memory[1] = 'error' + AttributeError: 'int' object has no attribute 'translate' + + .. versionadded:: 7.1 + + .. versionchanged:: 8.0 + + Moved from ``tools`` to ``tools.memories``. + + The parameter ``identifier_factory`` has been added to properly + transform ``str`` into :class:`~.identifiers.Identifier`. This factory + is stored and accessible through :attr:`make_identifier`. + + """ + def __init__( + self, + *args, + identifier_factory: IdentifierFactory = Identifier, + ) -> None: + super().__init__(*args) + self.make_identifier = identifier_factory + """A factory to transform keys into identifiers.""" + + def __getitem__(self, key): + return super().__getitem__(self.make_identifier(key)) + + def __contains__(self, key): + return super().__contains__(self.make_identifier(key)) + + def __setitem__(self, key, value): + super().__setitem__(self.make_identifier(key), value) diff --git a/sopel/tools/target.py b/sopel/tools/target.py index 97d698c9ac..c02dfa6973 100644 --- a/sopel/tools/target.py +++ b/sopel/tools/target.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, Optional, Set, Union from sopel import privileges -from sopel.tools import identifiers +from sopel.tools import identifiers, memories IdentifierFactory = Callable[[str], identifiers.Identifier] @@ -94,13 +94,32 @@ def __init__( assert isinstance(name, identifiers.Identifier) self.name = name """The name of the channel.""" - self.users: Dict[identifiers.Identifier, User] = {} + + self.make_identifier: IdentifierFactory = identifier_factory + """Factory to create :class:`~sopel.tools.identifiers.Identifier`. + + ``Identifier`` is used for :class:`User`'s nick, and the channel + needs to translate nicks from string to ``Identifier`` when + manipulating data associated to a user by its nickname. + """ + + self.users: Dict[ + identifiers.Identifier, + User, + ] = memories.SopelIdentifierMemory( + identifier_factory=self.make_identifier, + ) """The users in the channel. This maps nickname :class:`~sopel.tools.identifiers.Identifier`\\s to :class:`User` objects. """ - self.privileges: Dict[identifiers.Identifier, int] = {} + self.privileges: Dict[ + identifiers.Identifier, + int, + ] = memories.SopelIdentifierMemory( + identifier_factory=self.make_identifier, + ) """The permissions of the users in the channel. This maps nickname :class:`~sopel.tools.identifiers.Identifier`\\s to @@ -133,14 +152,6 @@ def __init__( is available, otherwise the time Sopel received it. """ - self.make_identifier: IdentifierFactory = identifier_factory - """Factory to create :class:`~sopel.tools.identifiers.Identifier`. - - ``Identifier`` is used for :class:`User`'s nick, and the channel - needs to translate nicks from string to ``Identifier`` when - manipulating data associated to a user by its nickname. - """ - def clear_user(self, nick: identifiers.Identifier) -> None: """Remove ``nick`` from this channel. diff --git a/test/test_tools.py b/test/test_tools.py index 196cc8b041..46a9e6ba88 100644 --- a/test/test_tools.py +++ b/test/test_tools.py @@ -118,54 +118,3 @@ def loader_text(settings): results = loader(settings) assert results == [re_numeric, re_text] - - -def test_sopel_identifier_memory_str(): - user = tools.Identifier('Exirel') - memory = tools.SopelIdentifierMemory() - test_value = 'king' - - memory['Exirel'] = test_value - assert user in memory - assert 'Exirel' in memory - assert 'exirel' in memory - assert 'exi' not in memory - assert '#channel' not in memory - - assert memory[user] == test_value - assert memory['Exirel'] == test_value - assert memory['exirel'] == test_value - - -def test_sopel_identifier_memory_id(): - user = tools.Identifier('Exirel') - memory = tools.SopelIdentifierMemory() - test_value = 'king' - - memory[user] = test_value - assert user in memory - assert 'Exirel' in memory - assert 'exirel' in memory - assert 'exi' not in memory - assert '#channel' not in memory - - assert memory[user] == test_value - assert memory['Exirel'] == test_value - assert memory['exirel'] == test_value - - -def test_sopel_identifier_memory_channel_str(): - channel = tools.Identifier('#adminchannel') - memory = tools.SopelIdentifierMemory() - test_value = 'perfect' - - memory['#adminchannel'] = test_value - assert channel in memory - assert '#adminchannel' in memory - assert '#AdminChannel' in memory - assert 'adminchannel' not in memory - assert 'Exirel' not in memory - - assert memory[channel] == test_value - assert memory['#adminchannel'] == test_value - assert memory['#AdminChannel'] == test_value diff --git a/test/tools/test_tools_memories.py b/test/tools/test_tools_memories.py new file mode 100644 index 0000000000..c0e49402f5 --- /dev/null +++ b/test/tools/test_tools_memories.py @@ -0,0 +1,55 @@ +"""Tests for Sopel Memory data-structures""" +from __future__ import annotations + +from sopel.tools import identifiers, memories + + +def test_sopel_identifier_memory_str(): + user = identifiers.Identifier('Exirel') + memory = memories.SopelIdentifierMemory() + test_value = 'king' + + memory['Exirel'] = test_value + assert user in memory + assert 'Exirel' in memory + assert 'exirel' in memory + assert 'exi' not in memory + assert '#channel' not in memory + + assert memory[user] == test_value + assert memory['Exirel'] == test_value + assert memory['exirel'] == test_value + + +def test_sopel_identifier_memory_id(): + user = identifiers.Identifier('Exirel') + memory = memories.SopelIdentifierMemory() + test_value = 'king' + + memory[user] = test_value + assert user in memory + assert 'Exirel' in memory + assert 'exirel' in memory + assert 'exi' not in memory + assert '#channel' not in memory + + assert memory[user] == test_value + assert memory['Exirel'] == test_value + assert memory['exirel'] == test_value + + +def test_sopel_identifier_memory_channel_str(): + channel = identifiers.Identifier('#adminchannel') + memory = memories.SopelIdentifierMemory() + test_value = 'perfect' + + memory['#adminchannel'] = test_value + assert channel in memory + assert '#adminchannel' in memory + assert '#AdminChannel' in memory + assert 'adminchannel' not in memory + assert 'Exirel' not in memory + + assert memory[channel] == test_value + assert memory['#adminchannel'] == test_value + assert memory['#AdminChannel'] == test_value