From 2df4ac04fb6e01c8581626525cfec62db69d7463 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 9 Feb 2023 16:14:05 +0100 Subject: [PATCH 01/13] Fixed last issue of #37 --- python_utils/loguru.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 7f172b6..c09fd4f 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -2,7 +2,7 @@ import loguru -from . import logger as logger_module +from . import logger as logger_module, types __all__ = ['Logurud'] @@ -10,6 +10,6 @@ class Logurud(logger_module.LoggerBase): logger: loguru.Logger - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: types.Any, **kwargs: types.Any) -> Logurud: cls.logger: loguru.Logger = loguru.logger.opt(depth=1) return super().__new__(cls) From 0ad935e2b6e30c6328ec97b9a698e87a065f71ba Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 13 Mar 2023 22:12:57 +0100 Subject: [PATCH 02/13] added security contact information --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 9924421..9573a7e 100644 --- a/README.rst +++ b/README.rst @@ -25,6 +25,13 @@ Links - Documentation: https://python-utils.readthedocs.io/en/latest/ - My blog: https://wol.ph/ +Security contact information +------------------------------------------------------------------------------ + +To report a security vulnerability, please use the +`Tidelift security contact `_. +Tidelift will coordinate the fix and disclosure. + Requirements for installing: ------------------------------------------------------------------------------ From 94273313cfa0229708966287e42d8eaa2e26cbdc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 21 May 2023 01:10:16 +0200 Subject: [PATCH 03/13] Applied black formatting --- docs/conf.py | 2 +- python_utils/formatters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9d82d43..63e2a44 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,6 +13,7 @@ from datetime import date import os import sys + sys.path.insert(0, os.path.abspath('..')) from python_utils import __about__ @@ -63,4 +64,3 @@ # html_static_path = ['_static'] intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} - diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 3449e3e..d2d76cb 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -44,7 +44,7 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( function: types.Callable[[str], str], data: types.OptionalScope = None, - **kwargs + **kwargs, ) -> types.OptionalScope: ''' Apply a function to all keys in a scope recursively From 5a200d87dec79ed263db2983b43ff834021be1cd Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 May 2023 17:16:42 +0200 Subject: [PATCH 04/13] Dropped Python 3.7 support and added full generic strict Pyright and Mypy compliant type hinting --- .gitignore | 1 + pyrightconfig.json | 8 ----- python_utils/aio.py | 12 ++++--- python_utils/containers.py | 55 +++++++++++++++++------------ python_utils/converters.py | 54 ++++++++++++++--------------- python_utils/decorators.py | 53 ++++++++++++++++++++-------- python_utils/exceptions.py | 14 ++++---- python_utils/formatters.py | 9 ++--- python_utils/generators.py | 28 ++++++++++----- python_utils/logger.py | 6 ++-- python_utils/loguru.py | 5 +-- python_utils/terminal.py | 39 ++++++--------------- python_utils/time.py | 71 +++++++++++++++++++++++++++++--------- python_utils/types.py | 19 +++------- setup.py | 2 +- 15 files changed, 212 insertions(+), 164 deletions(-) delete mode 100644 pyrightconfig.json diff --git a/.gitignore b/.gitignore index a04bcd0..46105bf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /docs/_build /cover /.eggs +/.* \ No newline at end of file diff --git a/pyrightconfig.json b/pyrightconfig.json deleted file mode 100644 index aaf3faf..0000000 --- a/pyrightconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "include": [ - "python_utils" - ], - "exclude": [ - "python_utils/types.py", - ], -} diff --git a/python_utils/aio.py b/python_utils/aio.py index de1ff87..ef17cb6 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -7,17 +7,19 @@ from . import types +_N = types.TypeVar('_N', int, float) + async def acount( - start: float = 0, - step: float = 1, + start: _N = 0, + step: _N = 1, delay: float = 0, - stop: types.Optional[float] = None, -) -> types.AsyncIterator[float]: + stop: types.Optional[_N] = None, +) -> types.AsyncIterator[_N]: '''Asyncio version of itertools.count()''' for item in itertools.count(start, step): # pragma: no branch if stop is not None and item >= stop: break - yield item + yield types.cast(_N, item) await asyncio.sleep(delay) diff --git a/python_utils/containers.py b/python_utils/containers.py index f8920a8..187b816 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,3 +1,4 @@ +# pyright: reportIncompatibleMethodOverride=false import abc import typing from typing import Any, Generator @@ -22,28 +23,32 @@ # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ - types.Mapping, - types.Iterable[types.Union[types.Tuple[Any, Any], types.Mapping]], + types.Mapping[types.Any, types.Any], + types.Iterable[ + types.Union[types.Tuple[Any, Any], types.Mapping[types.Any, types.Any]] + ], '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] class CastedDictBase(types.Dict[KT, VT], abc.ABC): - _key_cast: KT_cast - _value_cast: VT_cast + _key_cast: KT_cast[KT] + _value_cast: VT_cast[VT] def __init__( self, - key_cast: KT_cast = None, - value_cast: VT_cast = None, - *args: DictUpdateArgs, + key_cast: KT_cast[KT] = None, + value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], **kwargs: VT, ) -> None: self._value_cast = value_cast self._key_cast = key_cast self.update(*args, **kwargs) - def update(self, *args: DictUpdateArgs, **kwargs: VT) -> None: + def update( + self, *args: DictUpdateArgs[types.Any, types.Any], **kwargs: types.Any + ) -> None: if args: kwargs.update(*args) @@ -93,7 +98,7 @@ class CastedDict(CastedDictBase[KT, VT]): {1: 2, '3': '4', '5': '6', '7': '8'} ''' - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: if self._value_cast is not None: value = self._value_cast(value) @@ -146,13 +151,13 @@ class LazyCastedDict(CastedDictBase[KT, VT]): '4' ''' - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: types.Any, value: types.Any): if self._key_cast is not None: key = self._key_cast(key) super().__setitem__(key, value) - def __getitem__(self, key: Any) -> VT: + def __getitem__(self, key: types.Any) -> VT: if self._key_cast is not None: key = self._key_cast(key) @@ -163,9 +168,7 @@ def __getitem__(self, key: Any) -> VT: return value - def items( # type: ignore - self, - ) -> Generator[types.Tuple[KT, VT], None, None]: + def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore if self._value_cast is None: yield from super().items() else: @@ -247,7 +250,7 @@ def append(self, value: HT) -> None: self._set.add(value) super().append(value) - def __contains__(self, item): + def __contains__(self, item: HT) -> bool: # type: ignore return item in self._set @types.overload @@ -258,29 +261,37 @@ def __setitem__(self, indices: types.SupportsIndex, values: HT) -> None: def __setitem__(self, indices: slice, values: types.Iterable[HT]) -> None: ... - def __setitem__(self, indices, values) -> None: + def __setitem__( + self, + indices: types.Union[slice, types.SupportsIndex], + values: types.Union[types.Iterable[HT], HT], + ) -> None: if isinstance(indices, slice): + values = types.cast(types.Iterable[HT], values) if self.on_duplicate == 'ignore': raise RuntimeError( 'ignore mode while setting slices introduces ambiguous ' 'behaviour and is therefore not supported' ) - duplicates = set(values) & self._set - if duplicates and values != self[indices]: - raise ValueError('Duplicate values: %s' % duplicates) + duplicates: types.Set[HT] = set(values) & self._set + if duplicates and values != list(self[indices]): + raise ValueError(f'Duplicate values: {duplicates}') self._set.update(values) - super().__setitem__(indices, values) else: + values = types.cast(HT, values) if values in self._set and values != self[indices]: if self.on_duplicate == 'raise': - raise ValueError('Duplicate value: %s' % values) + raise ValueError(f'Duplicate value: {values}') else: return self._set.add(values) - super().__setitem__(indices, values) + + super().__setitem__( + types.cast(slice, indices), types.cast(types.List[HT], values) + ) def __delitem__( self, index: types.Union[types.SupportsIndex, slice] diff --git a/python_utils/converters.py b/python_utils/converters.py index c1eba73..e8c91f6 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -5,12 +5,14 @@ from . import types +_TN = types.TypeVar('_TN', bound=types.DecimalNumber) + def to_int( input_: typing.Optional[str] = None, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.Optional[types.Pattern] = None, + regexp: types.Optional[types.Pattern[str]] = None, ) -> int: r''' Convert the given input to an integer or return default @@ -84,8 +86,7 @@ def to_int( try: if regexp and input_: - match = regexp.search(input_) - if match: + if match := regexp.search(input_): input_ = match.groups()[-1] if input_ is None: @@ -100,7 +101,7 @@ def to_float( input_: str, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.Optional[types.Pattern] = None, + regexp: types.Optional[types.Pattern[str]] = None, ) -> types.Number: r''' Convert the given `input_` to an integer or return default @@ -165,8 +166,7 @@ def to_float( try: if regexp: - match = regexp.search(input_) - if match: + if match := regexp.search(input_): input_ = match.group(1) return float(input_) except exception: @@ -223,9 +223,7 @@ def to_str( >>> to_str(Foo) "" ''' - if isinstance(input_, bytes): - pass - else: + if not isinstance(input_, bytes): if not hasattr(input_, 'encode'): input_ = str(input_) @@ -263,12 +261,12 @@ def scale_1024( def remap( - value: types.DecimalNumber, - old_min: types.DecimalNumber, - old_max: types.DecimalNumber, - new_min: types.DecimalNumber, - new_max: types.DecimalNumber, -) -> types.DecimalNumber: + value: _TN, + old_min: _TN, + old_max: _TN, + new_min: _TN, + new_max: _TN, +) -> _TN: ''' remap a value from one range into another. @@ -362,24 +360,22 @@ def remap( else: type_ = int - value = type_(value) - old_min = type_(old_min) - old_max = type_(old_max) - new_max = type_(new_max) - new_min = type_(new_min) + value = types.cast(_TN, type_(value)) + old_min = types.cast(_TN, type_(old_min)) + old_max = types.cast(_TN, type_(old_max)) + new_max = types.cast(_TN, type_(new_max)) + new_min = types.cast(_TN, type_(new_min)) - old_range = old_max - old_min # type: ignore - new_range = new_max - new_min # type: ignore + # These might not be floats but the Python type system doesn't understand the + # generic type system in this case + old_range = types.cast(float, old_max) - types.cast(float, old_min) + new_range = types.cast(float, new_max) - types.cast(float, new_min) if old_range == 0: - raise ValueError( - 'Input range ({}-{}) is empty'.format(old_min, old_max) - ) + raise ValueError(f'Input range ({old_min}-{old_max}) is empty') if new_range == 0: - raise ValueError( - 'Output range ({}-{}) is empty'.format(new_min, new_max) - ) + raise ValueError(f'Output range ({new_min}-{new_max}) is empty') new_value = (value - old_min) * new_range # type: ignore @@ -390,4 +386,4 @@ def remap( new_value += new_min # type: ignore - return new_value + return types.cast(_TN, new_value) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 6c237aa..a5484e2 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -3,8 +3,12 @@ import random from . import types +T = types.TypeVar('T') +TC = types.TypeVar('TC', bound=types.Container[types.Any]) +P = types.ParamSpec('P') -def set_attributes(**kwargs): + +def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: '''Decorator to set attributes on functions and classes A common usage for this pattern is the Django Admin where @@ -28,7 +32,9 @@ def set_attributes(**kwargs): ''' - def _set_attributes(function): + def _set_attributes( + function: types.Callable[P, T] + ) -> types.Callable[P, T]: for key, value in kwargs.items(): setattr(function, key, value) return function @@ -36,7 +42,13 @@ def _set_attributes(function): return _set_attributes -def listify(collection: types.Callable = list, allow_empty: bool = True): +def listify( + collection: types.Callable[[types.Iterable[T]], TC] = list, # type: ignore + allow_empty: bool = True, +) -> types.Callable[ + [types.Callable[..., types.Optional[types.Iterable[T]]]], + types.Callable[..., TC], +]: ''' Convert any generator to a list or other type of collection. @@ -60,10 +72,10 @@ def listify(collection: types.Callable = list, allow_empty: bool = True): ... def empty_generator_not_allowed(): ... pass - >>> empty_generator_not_allowed() + >>> empty_generator_not_allowed() # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: 'NoneType' object is not iterable + TypeError: ... `allow_empty` is `False` >>> @listify(collection=set) ... def set_generator(): @@ -83,13 +95,22 @@ def listify(collection: types.Callable = list, allow_empty: bool = True): {'a': 1, 'b': 2} ''' - def _listify(function): - @functools.wraps(function) - def __listify(*args, **kwargs): - result = function(*args, **kwargs) - if result is None and allow_empty: - return [] - return collection(result) + def _listify( + function: types.Callable[..., types.Optional[types.Iterable[T]]] + ) -> types.Callable[..., TC]: + def __listify(*args: types.Any, **kwargs: types.Any) -> TC: + result: types.Optional[types.Iterable[T]] = function( + *args, **kwargs + ) + if result is None: + if allow_empty: + return collection(iter(())) + else: + raise TypeError( + f'{function} returned `None` and `allow_empty` is `False`' + ) + else: + return collection(result) return __listify @@ -109,12 +130,13 @@ def sample(sample_rate: float): ... return 1 Calls to *demo_function* will be limited to 50% approximatly. - ''' - def _sample(function): + def _sample( + function: types.Callable[P, T] + ) -> types.Callable[P, types.Optional[T]]: @functools.wraps(function) - def __sample(*args, **kwargs): + def __sample(*args: P.args, **kwargs: P.kwargs) -> types.Optional[T]: if random.random() < sample_rate: return function(*args, **kwargs) else: @@ -124,6 +146,7 @@ def __sample(*args, **kwargs): args, kwargs, ) # noqa: E501 + return None return __sample diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index 14855c5..3ffc01a 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -1,11 +1,11 @@ -import typing +from . import types def raise_exception( - exception_class: typing.Type[Exception], - *args: typing.Any, - **kwargs: typing.Any, -) -> typing.Callable: + exception_class: types.Type[Exception], + *args: types.Any, + **kwargs: types.Any, +) -> types.Callable[..., None]: ''' Returns a function that raises an exception of the given type with the given arguments. @@ -16,11 +16,11 @@ def raise_exception( ValueError: spam ''' - def raise_(*args_: typing.Any, **kwargs_: typing.Any) -> typing.Any: + def raise_(*args_: types.Any, **kwargs_: types.Any) -> types.Any: raise exception_class(*args, **kwargs) return raise_ -def reraise(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: +def reraise(*args: types.Any, **kwargs: types.Any) -> types.Any: raise diff --git a/python_utils/formatters.py b/python_utils/formatters.py index d2d76cb..04d661d 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,3 +1,4 @@ +# pyright: reportUnnecessaryIsInstance=false import datetime from python_utils import types @@ -21,7 +22,7 @@ def camel_to_underscore(name: str) -> str: >>> camel_to_underscore('__SpamANDBacon__') '__spam_and_bacon__' ''' - output = [] + output: types.List[str] = [] for i, c in enumerate(name): if i > 0: pc = name[i - 1] @@ -44,7 +45,7 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( function: types.Callable[[str], str], data: types.OptionalScope = None, - **kwargs, + **kwargs: types.Any, ) -> types.OptionalScope: ''' Apply a function to all keys in a scope recursively @@ -137,7 +138,7 @@ def timesince( (diff.seconds % 60, 'second', 'seconds'), ) - output = [] + output: types.List[str] = [] for period, singular, plural in periods: if int(period): if int(period) == 1: @@ -146,6 +147,6 @@ def timesince( output.append('%d %s' % (period, plural)) if output: - return '%s ago' % ' and '.join(output[:2]) + return f'{" and ".join(output[:2])} ago' return default diff --git a/python_utils/generators.py b/python_utils/generators.py index 2930334..b827ed4 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -5,16 +5,19 @@ from python_utils import types +_T = types.TypeVar('_T') + + async def abatcher( - generator: types.AsyncIterator, + generator: types.AsyncGenerator[_T, None], batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, -) -> types.AsyncIterator[list]: +) -> types.AsyncGenerator[types.List[_T], None]: ''' Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). ''' - batch: list = [] + batch: types.List[_T] = [] assert batch_size or interval, 'Must specify either batch_size or interval' @@ -26,16 +29,22 @@ async def abatcher( # Set the timeout to 10 years interval_s = 60 * 60 * 24 * 365 * 10.0 - next_yield = time.perf_counter() + interval_s + next_yield: float = time.perf_counter() + interval_s - pending: types.Set = set() + done: types.Set[asyncio.Task[_T]] + pending: types.Set[asyncio.Task[_T]] = set() while True: try: done, pending = await asyncio.wait( pending or [ - asyncio.create_task(generator.__anext__()), # type: ignore + asyncio.create_task( + types.cast( + types.Coroutine[None, None, _T], + generator.__anext__(), + ) + ), ], timeout=interval_s, return_when=asyncio.FIRST_COMPLETED, @@ -65,12 +74,13 @@ async def abatcher( def batcher( - iterable: types.Iterable, batch_size: int = 10 -) -> types.Iterator[list]: + iterable: types.Iterable[_T], + batch_size: int = 10, +) -> types.Generator[types.List[_T], None, None]: ''' Generator wrapper that returns items with a given batch size ''' - batch = [] + batch: types.List[_T] = [] for item in iterable: batch.append(item) if len(batch) == batch_size: diff --git a/python_utils/logger.py b/python_utils/logger.py index 10ee5e6..bcf0bab 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -33,7 +33,9 @@ class LoggerBase(abc.ABC): logger: typing.Any @classmethod - def __get_name(cls, *name_parts: str) -> str: + def __get_name( # pyright: ignore[reportUnusedFunction] + cls, *name_parts: str + ) -> str: return '.'.join(n.strip() for n in name_parts if n.strip()) @classmethod @@ -95,7 +97,7 @@ class Logged(LoggerBase): def __get_name(cls, *name_parts: str) -> str: return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: typing.Any, **kwargs: typing.Any): cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index c09fd4f..22b258d 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -1,8 +1,9 @@ from __future__ import annotations +import typing import loguru -from . import logger as logger_module, types +from . import logger as logger_module __all__ = ['Logurud'] @@ -10,6 +11,6 @@ class Logurud(logger_module.LoggerBase): logger: loguru.Logger - def __new__(cls, *args: types.Any, **kwargs: types.Any) -> Logurud: + def __new__(cls, *args: typing.Any, **kwargs: typing.Any): cls.logger: loguru.Logger = loguru.logger.opt(depth=1) return super().__new__(cls) diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 19c494d..53948d8 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -1,3 +1,4 @@ +import contextlib import os import typing @@ -20,7 +21,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover w: typing.Optional[int] h: typing.Optional[int] - try: + with contextlib.suppress(Exception): # Default to 79 characters for IPython notebooks from IPython import get_ipython # type: ignore @@ -29,10 +30,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover if isinstance(ipython, zmqshell.ZMQInteractiveShell): return 79, 24 - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): # This works for Python 3, but not Pypy3. Probably the best method if # it's supported so let's always try import shutil @@ -42,18 +40,12 @@ def get_terminal_size() -> Dimensions: # pragma: no cover # The off by one is needed due to progressbars in some cases, for # safety we'll always substract it. return w - 1, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): w = converters.to_int(os.environ.get('COLUMNS')) h = converters.to_int(os.environ.get('LINES')) if w and h: return w, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): import blessings # type: ignore terminal = blessings.Terminal() @@ -61,32 +53,23 @@ def get_terminal_size() -> Dimensions: # pragma: no cover h = terminal.height if w and h: return w, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): # The method can return None so we don't unpack it wh = _get_terminal_size_linux() if wh is not None and all(wh): return wh - except Exception: # pragma: no cover - pass - try: + with contextlib.suppress(Exception): # Windows detection doesn't always work, let's try anyhow wh = _get_terminal_size_windows() if wh is not None and all(wh): return wh - except Exception: # pragma: no cover - pass - try: + with contextlib.suppress(Exception): # needed for window's python in cygwin's xterm! wh = _get_terminal_size_tput() if wh is not None and all(wh): return wh - except Exception: # pragma: no cover - pass return 79, 24 @@ -162,12 +145,10 @@ def ioctl_GWINSZ(fd): size = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) if not size: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) + with contextlib.suppress(Exception): + fd = os.open(os.ctermid(), os.O_RDONLY) # type: ignore size = ioctl_GWINSZ(fd) os.close(fd) - except Exception: - pass if not size: try: size = os.environ['LINES'], os.environ['COLUMNS'] diff --git a/python_utils/time.py b/python_utils/time.py index ce1d91c..4084a5a 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,3 +1,4 @@ +# pyright: reportUnnecessaryIsInstance=false import asyncio import datetime import functools @@ -7,6 +8,10 @@ import python_utils from python_utils import aio, exceptions, types +_T = types.TypeVar('_T') +_P = types.ParamSpec('_P') + + # There might be a better way to get the epoch with tzinfo, please create # a pull request if you know a better way that functions for Python 2 and 3 epoch = datetime.datetime(year=1970, month=1, day=1) @@ -139,7 +144,9 @@ def format_time( def timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[types.Iterable, types.Callable] = itertools.count, + iterable: types.Union[ + types.Iterable[_T], types.Callable[[], types.Iterable[_T]] + ] = itertools.count, # type: ignore interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ): @@ -179,7 +186,7 @@ def timeout_generator( maximum_interval ) - iterable_: types.Iterable + iterable_: types.Iterable[_T] if callable(iterable): iterable_ = iterable() else: @@ -202,12 +209,14 @@ def timeout_generator( async def aio_timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[types.AsyncIterable, types.Callable] = aio.acount, + iterable: types.Union[ + types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]] + ] = aio.acount, interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, -): +) -> types.AsyncGenerator[_T, None]: ''' - Aync generator that walks through the given iterable (a counter by + Async generator that walks through the given async iterable (a counter by default) until the float_timeout is reached with a configurable float_interval between items @@ -226,7 +235,7 @@ async def aio_timeout_generator( maximum_interval ) - iterable_: types.AsyncIterable + iterable_: types.AsyncIterable[_T] if callable(iterable): iterable_ = iterable() else: @@ -247,12 +256,22 @@ async def aio_timeout_generator( async def aio_generator_timeout_detector( - generator: types.AsyncGenerator, + generator: types.AsyncGenerator[_T, None], timeout: types.Optional[types.delta_type] = None, total_timeout: types.Optional[types.delta_type] = None, - on_timeout: types.Optional[types.Callable] = exceptions.reraise, - **kwargs, -): + on_timeout: types.Optional[ + types.Callable[ + [ + types.AsyncGenerator[_T, None], + types.Optional[types.delta_type], + types.Optional[types.delta_type], + BaseException, + ], + types.Any, + ] + ] = exceptions.reraise, + **on_timeout_kwargs: types.Mapping[types.Text, types.Any], +) -> types.AsyncGenerator[_T, None]: ''' This function is used to detect if an asyncio generator has not yielded an element for a set amount of time. @@ -286,7 +305,11 @@ async def aio_generator_timeout_detector( except asyncio.TimeoutError as exception: if on_timeout is not None: await on_timeout( - generator, timeout, total_timeout, exception, **kwargs + generator, + timeout, + total_timeout, + exception, + **on_timeout_kwargs, ) break @@ -297,26 +320,40 @@ async def aio_generator_timeout_detector( def aio_generator_timeout_detector_decorator( timeout: types.Optional[types.delta_type] = None, total_timeout: types.Optional[types.delta_type] = None, - on_timeout: types.Optional[types.Callable] = exceptions.reraise, - **kwargs, + on_timeout: types.Optional[ + types.Callable[ + [ + types.AsyncGenerator[types.Any, None], + types.Optional[types.delta_type], + types.Optional[types.delta_type], + BaseException, + ], + types.Any, + ] + ] = exceptions.reraise, + **on_timeout_kwargs: types.Mapping[types.Text, types.Any], ): ''' A decorator wrapper for aio_generator_timeout_detector. ''' - def _timeout_detector_decorator(generator: types.Callable): + def _timeout_detector_decorator( + generator: types.Callable[_P, types.AsyncGenerator[_T, None]] + ) -> types.Callable[_P, types.AsyncGenerator[_T, None]]: ''' The decorator itself. ''' @functools.wraps(generator) - def wrapper(*args, **wrapper_kwargs): + def wrapper( + *args: _P.args, **kwargs: _P.kwargs + ) -> types.AsyncGenerator[_T, None]: return aio_generator_timeout_detector( - generator(*args, **wrapper_kwargs), + generator(*args, **kwargs), timeout, total_timeout, on_timeout, - **kwargs, + **on_timeout_kwargs, ) return wrapper diff --git a/python_utils/types.py b/python_utils/types.py index 1e749c8..1dfcd63 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,15 +1,11 @@ +# pyright: reportWildcardImportFromLibrary=false import datetime import decimal -import sys -from typing import * # type: ignore # pragma: no cover +from typing_extensions import * # type: ignore # pragma: no cover # noqa: F403 +from typing import * # type: ignore # pragma: no cover # noqa: F403 -# import * does not import Pattern -from typing import Pattern - -if sys.version_info >= (3, 8): # pragma: no cover - from typing import Literal, SupportsIndex -else: # pragma: no cover - from typing_extensions import Literal, SupportsIndex +# import * does not import these in all Python versions +from typing import Pattern, BinaryIO, IO, TextIO, Match # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of @@ -39,10 +35,6 @@ None, ] -assert Pattern is not None # type: ignore -assert Literal is not None -assert SupportsIndex is not None - __all__ = [ 'OptionalScope', 'Number', @@ -141,5 +133,4 @@ 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', - 'TracebackType', ] diff --git a/setup.py b/setup.py index af90617..b5ae708 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ ), package_data={'python_utils': ['py.typed']}, long_description=long_description, - install_requires=['typing_extensions;python_version<"3.8"'], + install_requires=['typing_extensions'], tests_require=['pytest'], extras_require={ 'loguru': [ From f96fbe0b2b3b52b7eac91c6e02d0ed210a681750 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:02:07 +0200 Subject: [PATCH 05/13] Added AIO tests --- _python_utils_tests/test_aio.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 _python_utils_tests/test_aio.py diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py new file mode 100644 index 0000000..3e275cd --- /dev/null +++ b/_python_utils_tests/test_aio.py @@ -0,0 +1,20 @@ +from datetime import datetime +import pytest +import asyncio +from python_utils.aio import acount + + +@pytest.mark.asyncio +async def test_acount(monkeypatch: pytest.MonkeyPatch): + sleeps: list[float] = [] + + async def mock_sleep(delay: float): + sleeps.append(delay) + + monkeypatch.setattr(asyncio, 'sleep', mock_sleep) + + async for i in acount(delay=1, stop=3.5): + print('i', i, datetime.now()) + + assert len(sleeps) == 4 + assert sum(sleeps) == 4 From 4f14340ae50f974dfe049c9e4c24333ffe055a83 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:02:37 +0200 Subject: [PATCH 06/13] Added pyproject.toml for black and pyright configuration --- pyproject.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2720bf2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.black] +line-length = 79 +target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] +skip-string-normalization = true + +[tool.pyright] +include = ['python_utils'] +strict = ['python_utils', '_python_utils_tests/test_aio.py'] +# The terminal file is very OS specific and dependent on imports so we're skipping it from type checking +ignore = ['python_utils/terminal.py'] +pythonVersion = '3.8' \ No newline at end of file From c7a3c467a285b41aae5465535563ce859480d66f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:28:20 +0200 Subject: [PATCH 07/13] Added types from the `types` module --- python_utils/types.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/python_utils/types.py b/python_utils/types.py index 1dfcd63..cd63b53 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -3,6 +3,7 @@ import decimal from typing_extensions import * # type: ignore # pragma: no cover # noqa: F403 from typing import * # type: ignore # pragma: no cover # noqa: F403 +from types import * # type: ignore # pragma: no cover # noqa: F403 # import * does not import these in all Python versions from typing import Pattern, BinaryIO, IO, TextIO, Match @@ -55,13 +56,15 @@ 'SupportsIndex', 'Optional', 'ParamSpec', + 'ParamSpecArgs', + 'ParamSpecKwargs', 'Protocol', 'Tuple', 'Type', 'TypeVar', 'Union', # ABCs (from collections.abc). - 'AbstractSet', # collections.abc.Set. + 'AbstractSet', 'ByteString', 'Container', 'ContextManager', @@ -126,11 +129,36 @@ 'no_type_check_decorator', 'NoReturn', 'overload', - 'ParamSpecArgs', - 'ParamSpecKwargs', 'runtime_checkable', 'Text', 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TracebackType', + # Types from the `types` module. + 'FunctionType', + 'LambdaType', + 'CodeType', + 'MappingProxyType', + 'SimpleNamespace', + 'GeneratorType', + 'CoroutineType', + 'AsyncGeneratorType', + 'MethodType', + 'BuiltinFunctionType', + 'BuiltinMethodType', + 'WrapperDescriptorType', + 'MethodWrapperType', + 'MethodDescriptorType', + 'ClassMethodDescriptorType', + 'ModuleType', + 'TracebackType', + 'FrameType', + 'GetSetDescriptorType', + 'MemberDescriptorType', + 'new_class', + 'resolve_bases', + 'prepare_class', + 'DynamicClassAttribute', + 'coroutine', ] From e43a51279ccca32c001933d6915903578595808b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:28:34 +0200 Subject: [PATCH 08/13] Fixed tests and more Python version compatibility --- _python_utils_tests/test_aio.py | 4 +++- pytest.ini | 2 +- python_utils/containers.py | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 3e275cd..0eb95d3 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -1,12 +1,14 @@ from datetime import datetime import pytest import asyncio + +from python_utils import types from python_utils.aio import acount @pytest.mark.asyncio async def test_acount(monkeypatch: pytest.MonkeyPatch): - sleeps: list[float] = [] + sleeps: types.List[float] = [] async def mock_sleep(delay: float): sleeps.append(delay) diff --git a/pytest.ini b/pytest.ini index 5d49701..a8e632a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,7 +7,7 @@ addopts = --doctest-modules --cov python_utils --cov-report term-missing - --mypy +; --mypy doctest_optionflags = ALLOW_UNICODE diff --git a/python_utils/containers.py b/python_utils/containers.py index 187b816..31700e5 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,7 +1,6 @@ # pyright: reportIncompatibleMethodOverride=false import abc import typing -from typing import Any, Generator from . import types @@ -15,18 +14,17 @@ #: A type alias for a dictionary with keys of type KT and values of type VT. DT = types.Dict[KT, VT] #: A type alias for the casted type of a dictionary key. -KT_cast = types.Optional[types.Callable[[Any], KT]] +KT_cast = types.Optional[types.Callable[..., KT]] #: A type alias for the casted type of a dictionary value. -VT_cast = types.Optional[types.Callable[[Any], VT]] +VT_cast = types.Optional[types.Callable[..., VT]] #: A type alias for the hashable values of the `UniqueList` HT = types.TypeVar('HT', bound=types.Hashable) # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ - types.Mapping[types.Any, types.Any], - types.Iterable[ - types.Union[types.Tuple[Any, Any], types.Mapping[types.Any, types.Any]] - ], + types.Mapping[KT, VT], + types.Iterable[types.Tuple[KT, VT]], + types.Iterable[types.Mapping[KT, VT]], '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] @@ -56,7 +54,7 @@ def update( for key, value in kwargs.items(): self[key] = value - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: types.Any, value: types.Any) -> None: if self._key_cast is not None: key = self._key_cast(key) @@ -168,14 +166,16 @@ def __getitem__(self, key: types.Any) -> VT: return value - def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore + def items( # type: ignore + self, + ) -> types.Generator[types.Tuple[KT, VT], None, None]: if self._value_cast is None: yield from super().items() else: for key, value in super().items(): yield key, self._value_cast(value) - def values(self) -> Generator[VT, None, None]: # type: ignore + def values(self) -> types.Generator[VT, None, None]: # type: ignore if self._value_cast is None: yield from super().values() else: From 0592a5fbb62aa7e410f487fc631d2a98ee8c74e1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 May 2023 18:15:46 +0200 Subject: [PATCH 09/13] Added sliceable deque --- python_utils/containers.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/python_utils/containers.py b/python_utils/containers.py index 31700e5..53dcd57 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -305,6 +305,38 @@ def __delitem__( super().__delitem__(index) +class SlicableDeque(Generic[T], deque): + def __getitem__(self, index: Union[int, slice]) -> Union[T, 'SlicableDeque[T]']: + """ + Return the item or slice at the given index. + + >>> d = SlicableDeque[int]([1, 2, 3, 4, 5]) + >>> d[1:4] + SlicableDeque([2, 3, 4]) + + >>> d = SlicableDeque[str](['a', 'b', 'c']) + >>> d[-2:] + SlicableDeque(['b', 'c']) + + """ + if isinstance(index, slice): + start, stop, step = index.indices(len(self)) + return self.__class__(self[i] for i in range(start, stop, step)) + else: + return super().__getitem__(index) + + def pop(self) -> T: + """ + Remove and return the rightmost element. + + >>> d = SlicableDeque[float]([1.5, 2.5, 3.5]) + >>> d.pop() + 3.5 + + """ + return super().pop() + + if __name__ == '__main__': import doctest From 5eac0d3297187fea1194a04da5e26c1dabe60052 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:02:04 +0200 Subject: [PATCH 10/13] Added type hinting to new container classes --- python_utils/containers.py | 38 +++++++++++++++++++++----------------- python_utils/converters.py | 4 ++-- python_utils/decorators.py | 3 ++- python_utils/types.py | 2 +- tox.ini | 4 +--- 5 files changed, 27 insertions(+), 24 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 53dcd57..4685ab9 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,6 +1,7 @@ # pyright: reportIncompatibleMethodOverride=false import abc import typing +import collections from . import types @@ -19,6 +20,8 @@ VT_cast = types.Optional[types.Callable[..., VT]] #: A type alias for the hashable values of the `UniqueList` HT = types.TypeVar('HT', bound=types.Hashable) +#: A type alias for a regular generic type +T = types.TypeVar('T') # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ @@ -28,6 +31,8 @@ '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] +OnDuplicate = types.Literal['ignore', 'raise'] + class CastedDictBase(types.Dict[KT, VT], abc.ABC): _key_cast: KT_cast[KT] @@ -222,7 +227,7 @@ class UniqueList(types.List[HT]): def __init__( self, *args: HT, - on_duplicate: types.Literal['raise', 'ignore'] = 'ignore', + on_duplicate: OnDuplicate = 'ignore', ): self.on_duplicate = on_duplicate self._set = set() @@ -305,9 +310,19 @@ def __delitem__( super().__delitem__(index) -class SlicableDeque(Generic[T], deque): - def __getitem__(self, index: Union[int, slice]) -> Union[T, 'SlicableDeque[T]']: - """ +class SlicableDeque(types.Generic[T], collections.deque): # type: ignore + @types.overload + def __getitem__(self, index: types.SupportsIndex) -> T: + ... + + @types.overload + def __getitem__(self, index: slice) -> 'SlicableDeque[T]': + ... + + def __getitem__( + self, index: types.Union[types.SupportsIndex, slice] + ) -> types.Union[T, 'SlicableDeque[T]']: + ''' Return the item or slice at the given index. >>> d = SlicableDeque[int]([1, 2, 3, 4, 5]) @@ -318,23 +333,12 @@ def __getitem__(self, index: Union[int, slice]) -> Union[T, 'SlicableDeque[T]']: >>> d[-2:] SlicableDeque(['b', 'c']) - """ + ''' if isinstance(index, slice): start, stop, step = index.indices(len(self)) return self.__class__(self[i] for i in range(start, stop, step)) else: - return super().__getitem__(index) - - def pop(self) -> T: - """ - Remove and return the rightmost element. - - >>> d = SlicableDeque[float]([1.5, 2.5, 3.5]) - >>> d.pop() - 3.5 - - """ - return super().pop() + return types.cast(T, super().__getitem__(index)) if __name__ == '__main__': diff --git a/python_utils/converters.py b/python_utils/converters.py index e8c91f6..68438ee 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -366,8 +366,8 @@ def remap( new_max = types.cast(_TN, type_(new_max)) new_min = types.cast(_TN, type_(new_min)) - # These might not be floats but the Python type system doesn't understand the - # generic type system in this case + # These might not be floats but the Python type system doesn't understand + # the generic type system in this case old_range = types.cast(float, old_max) - types.cast(float, old_min) new_range = types.cast(float, new_max) - types.cast(float, new_min) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index a5484e2..6559cf0 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -107,7 +107,8 @@ def __listify(*args: types.Any, **kwargs: types.Any) -> TC: return collection(iter(())) else: raise TypeError( - f'{function} returned `None` and `allow_empty` is `False`' + f'{function} returned `None` and `allow_empty` ' + 'is `False`' ) else: return collection(result) diff --git a/python_utils/types.py b/python_utils/types.py index cd63b53..01c319a 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,7 +1,7 @@ # pyright: reportWildcardImportFromLibrary=false import datetime import decimal -from typing_extensions import * # type: ignore # pragma: no cover # noqa: F403 +from typing_extensions import * # type: ignore # noqa: F403 from typing import * # type: ignore # pragma: no cover # noqa: F403 from types import * # type: ignore # pragma: no cover # noqa: F403 diff --git a/tox.ini b/tox.ini index 8290500..f017232 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,13 @@ [tox] -envlist = black, py37, py38, py39, py310, py311, pypy3, flake8, docs, mypy, pyright +envlist = black, py38, py39, py310, py311, flake8, docs, mypy, pyright skip_missing_interpreters = True [testenv] basepython = - py37: python3.7 py38: python3.8 py39: python3.9 py310: python3.10 py311: python3.11 - pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 deps = From 95b5d3a5e84074e46345e5b1f414e446d9e28ac0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:11:31 +0200 Subject: [PATCH 11/13] dropped python 3.7 from tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b7d08dc..2f8ac44 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] + python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v3 From f50712a2856ae123f68b697eacbaa2d140813d9a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:57:31 +0200 Subject: [PATCH 12/13] Dropped Python 3.7 support --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b5ae708..4b14c0b 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ if __name__ == '__main__': setuptools.setup( - python_requires='>3.6.0', + python_requires='>3.8.0', name='python-utils', version=about['__version__'], author=about['__author__'], From e68d6e4fdfeb9fe757fd8cb73c88fe745aa7003f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:58:48 +0200 Subject: [PATCH 13/13] Incrementing version to v3.6.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 7e57bd8..69bddae 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.5.2' +__version__ = '3.6.0'