From 764e876b8af759f3d6beb35f3e7ff4d57213f253 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Thu, 4 Feb 2021 19:59:51 -0800 Subject: [PATCH] refactor versioning to be more easily extensible --- reorder_python_imports.py | 230 ++++++++++++++------------- testing/generate-future-info | 38 +++++ testing/generate-mock-info | 4 +- testing/generate-python-future-info | 28 ++++ testing/generate-six-info | 4 +- testing/generate-typing-rewrite-info | 78 ++++----- 6 files changed, 216 insertions(+), 166 deletions(-) create mode 100755 testing/generate-future-info create mode 100755 testing/generate-python-future-info diff --git a/reorder_python_imports.py b/reorder_python_imports.py index 2de6cf9..e9f8012 100644 --- a/reorder_python_imports.py +++ b/reorder_python_imports.py @@ -10,6 +10,7 @@ import tokenize from typing import Any from typing import Callable +from typing import Dict from typing import Generator from typing import Iterable from typing import List @@ -519,86 +520,102 @@ def _report_diff(contents: str, new_contents: str, filename: str) -> None: print(diff, end='') -FUTURE_IMPORTS: Tuple[Tuple[str, Tuple[str, ...]], ...] = ( - ('py22', ('nested_scopes',)), - ('py23', ('generators',)), - ('py26', ('with_statement',)), - ( - 'py3', - ('division', 'absolute_import', 'print_function', 'unicode_literals'), - ), - ('py37', ('generator_stop',)), -) +REMOVALS: Dict[Tuple[int, ...], Set[str]] = collections.defaultdict(set) +REPLACES: Dict[Tuple[int, ...], Set[str]] = collections.defaultdict(set) + +REMOVALS[(3,)].add('from io import open') + +# GENERATED VIA generate-future-info +REMOVALS[(2, 2)].add('from __future__ import nested_scopes') +REMOVALS[(2, 3)].add('from __future__ import generators') +REMOVALS[(2, 6)].add('from __future__ import with_statement') +REMOVALS[(3,)].add('from __future__ import absolute_import, division, print_function, unicode_literals') # noqa: E501 +REMOVALS[(3, 7)].add('from __future__ import generator_stop') +REMOVALS[(3, 10)].add('from __future__ import annotations') +# END GENERATED + # GENERATED VIA generate-typing-rewrite-info # Using: -# flake8-typing-imports==1.9.0 +# flake8-typing-imports==1.10.1 # mypy_extensions==0.4.3 -# typing_extensions==3.7.4.2 -MYPY_EXTENSIONS_IMPORTS: Tuple[Tuple[str, Tuple[str, ...]], ...] = ( - ('py37', ('NoReturn',)), - ('py38', ('TypedDict',)), -) -TYPING_EXTENSIONS_IMPORTS: Tuple[Tuple[str, Tuple[str, ...]], ...] = ( - ( - 'py36', ( - 'AsyncIterable', 'AsyncIterator', 'Awaitable', 'ClassVar', - 'ContextManager', 'Coroutine', 'DefaultDict', 'NewType', - 'TYPE_CHECKING', 'Text', 'Type', 'overload', - ), - ), - ( - 'py37', ( - 'AsyncContextManager', 'AsyncGenerator', 'ChainMap', 'Counter', - 'Deque', - ), - ), - ( - 'py38', ( - 'Final', 'Literal', 'Protocol', 'TypedDict', 'final', - 'runtime_checkable', - ), - ), -) +# typing_extensions==3.7.4.3 +REPLACES[(3, 7)].update(( + 'mypy_extensions=typing:NoReturn', +)) +REPLACES[(3, 8)].update(( + 'mypy_extensions=typing:TypedDict', +)) +REPLACES[(3, 6)].update(( + 'typing_extensions=typing:AsyncIterable', + 'typing_extensions=typing:AsyncIterator', + 'typing_extensions=typing:Awaitable', + 'typing_extensions=typing:ClassVar', + 'typing_extensions=typing:ContextManager', + 'typing_extensions=typing:Coroutine', + 'typing_extensions=typing:DefaultDict', + 'typing_extensions=typing:NewType', + 'typing_extensions=typing:TYPE_CHECKING', + 'typing_extensions=typing:Text', + 'typing_extensions=typing:Type', + 'typing_extensions=typing:get_type_hints', + 'typing_extensions=typing:overload', +)) +REPLACES[(3, 7)].update(( + 'typing_extensions=typing:AsyncContextManager', + 'typing_extensions=typing:AsyncGenerator', + 'typing_extensions=typing:ChainMap', + 'typing_extensions=typing:Counter', + 'typing_extensions=typing:Deque', +)) +REPLACES[(3, 8)].update(( + 'typing_extensions=typing:Final', + 'typing_extensions=typing:Literal', + 'typing_extensions=typing:Protocol', + 'typing_extensions=typing:SupportsIndex', + 'typing_extensions=typing:TypedDict', + 'typing_extensions=typing:final', + 'typing_extensions=typing:get_args', + 'typing_extensions=typing:get_origin', + 'typing_extensions=typing:runtime_checkable', +)) +REPLACES[(3, 9)].update(( + 'typing_extensions=typing:Annotated', +)) # END GENERATED - -BUILTINS = ( # from python-future - 'ascii', 'bytes', 'chr', 'dict', 'filter', 'hex', 'input', 'int', 'list', - 'map', 'max', 'min', 'next', 'object', 'oct', 'open', 'pow', 'range', - 'round', 'str', 'super', 'zip', -) - - -def _add_version_options(parser: argparse.ArgumentParser) -> None: - msg = 'Removes/updates obsolete imports; implies all older versions.' - - for py in sorted({ - py for py, _ in - FUTURE_IMPORTS + MYPY_EXTENSIONS_IMPORTS + TYPING_EXTENSIONS_IMPORTS - }): - parser.add_argument(f'--{py}-plus', action='store_true', help=msg) - - -def _version_removals(args: argparse.Namespace) -> Generator[str, None, None]: - implied = False - to_remove: List[str] = [] - for py, removals in reversed(FUTURE_IMPORTS): - implied |= getattr(args, f'{py}_plus') - if implied: - to_remove.extend(removals) - if to_remove: - yield f'from __future__ import {", ".join(to_remove)}' - - if _is_py3(args): - yield from SIX_REMOVALS - yield 'from io import open' - yield 'from builtins import *' - yield f'from builtins import {", ".join(BUILTINS)}' - +# GENERATED VIA generate-python-future-info +# Using future==0.18.2 +REMOVALS[(3,)].update(( + 'from builtins import *', + 'from builtins import ascii', + 'from builtins import bytes', + 'from builtins import chr', + 'from builtins import dict', + 'from builtins import filter', + 'from builtins import hex', + 'from builtins import input', + 'from builtins import int', + 'from builtins import isinstance', + 'from builtins import list', + 'from builtins import map', + 'from builtins import max', + 'from builtins import min', + 'from builtins import next', + 'from builtins import object', + 'from builtins import oct', + 'from builtins import open', + 'from builtins import pow', + 'from builtins import range', + 'from builtins import round', + 'from builtins import str', + 'from builtins import super', + 'from builtins import zip', +)) +# END GENERATED # GENERATED VIA generate-six-info -# Using six==1.14.0 -SIX_REMOVALS = [ +# Using six==1.15.0 +REMOVALS[(3,)].update(( 'from six import callable', 'from six import next', 'from six.moves import filter', @@ -606,8 +623,8 @@ def _version_removals(args: argparse.Namespace) -> Generator[str, None, None]: 'from six.moves import map', 'from six.moves import range', 'from six.moves import zip', -] -SIX_RENAMES = [ +)) +REPLACES[(3,)].update(( 'six.moves.BaseHTTPServer=http.server', 'six.moves.CGIHTTPServer=http.server', 'six.moves.SimpleHTTPServer=http.server', @@ -672,13 +689,12 @@ def _version_removals(args: argparse.Namespace) -> Generator[str, None, None]: 'six=functools:wraps', 'six=io:BytesIO', 'six=io:StringIO', -] +)) # END GENERATED - # GENERATED VIA generate-mock-info -# Using mock==4.0.2 -MOCK_RENAMES = [ +# Using mock==4.0.3 +REPLACES[(3,)].update(( 'mock.mock=unittest.mock:ANY', 'mock.mock=unittest.mock:DEFAULT', 'mock.mock=unittest.mock:FILTER_DIR', @@ -705,42 +721,24 @@ def _version_removals(args: argparse.Namespace) -> Generator[str, None, None]: 'mock=unittest.mock:mock_open', 'mock=unittest.mock:patch', 'mock=unittest.mock:sentinel', -] +)) # END GENERATED -def _is_py3(args: argparse.Namespace) -> bool: - pys = { - py for - py, _ in - FUTURE_IMPORTS + MYPY_EXTENSIONS_IMPORTS + TYPING_EXTENSIONS_IMPORTS - } +def _add_version_options(parser: argparse.ArgumentParser) -> None: + versions = sorted(REMOVALS.keys() | REPLACES.keys()) - return any( - py.startswith('py3') and getattr(args, f'{py}_plus') - for py in pys + msg = 'Removes/updates obsolete imports; implies all older versions.' + parser.add_argument( + f'--py{"".join(str(n) for n in versions[0])}-plus', help=msg, + action='store_const', dest='min_version', const=versions[0], + default=(0,), ) - - -def _typing_replaces(args: argparse.Namespace) -> Generator[str, None, None]: - for orig, imports in ( - ('mypy_extensions', MYPY_EXTENSIONS_IMPORTS), - ('typing_extensions', TYPING_EXTENSIONS_IMPORTS), - ): - implied = False - for py, attrs in reversed(imports): - implied |= getattr(args, f'{py}_plus') - if implied: - for attr in attrs: - yield f'{orig}=typing:{attr}' - - -def _version_replaces(args: argparse.Namespace) -> List[ImportToReplace]: - if _is_py3(args): - renames = MOCK_RENAMES + SIX_RENAMES + list(_typing_replaces(args)) - return [_validate_replace_import(s) for s in renames] - else: - return [] + for version in versions[1:]: + parser.add_argument( + f'--py{"".join(str(n) for n in version)}-plus', help=msg, + action='store_const', dest='min_version', const=version, + ) def _validate_import(s: str) -> str: @@ -843,8 +841,16 @@ def main(argv: Optional[Sequence[str]] = None) -> int: _add_version_options(parser) args = parser.parse_args(argv) - args.remove_import.extend(_version_removals(args)) - args.replace_import.extend(_version_replaces(args)) + + for k, v in REMOVALS.items(): + if args.min_version >= k: + args.remove_import.extend(v) + + for k, v in REPLACES.items(): + if args.min_version >= k: + args.replace_import.extend( + _validate_replace_import(replace_s) for replace_s in v + ) if os.environ.get('PYTHONPATH'): sys.stderr.write('$PYTHONPATH set, import order may be unexpected\n') diff --git a/testing/generate-future-info b/testing/generate-future-info new file mode 100755 index 0000000..b665a88 --- /dev/null +++ b/testing/generate-future-info @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import __future__ + +import collections +import os.path +import sys + + +def main() -> int: + assert sys.version_info >= (3, 10), 'need python3.10' + + by_version = collections.defaultdict(list) + + for k, v in vars(__future__).items(): + if k == 'barry_as_FLUFL' or not hasattr(v, 'mandatory'): + continue + + if v.mandatory[1] == 0: + version = (v.mandatory[0],) + else: + version = v.mandatory[:2] + + by_version[version].append(k) + + print(f'# GENERATED VIA {os.path.basename(sys.argv[0])}') + for v_k, v in sorted(by_version.items()): + s = f'from __future__ import {", ".join(sorted(v))}' + line = f'REMOVALS[{v_k}].add({s!r})' + if len(line) >= 79: + line = f'{line} # noqa: E501' + print(line) + print('# END GENERATED') + + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/testing/generate-mock-info b/testing/generate-mock-info index 475aa83..3958448 100755 --- a/testing/generate-mock-info +++ b/testing/generate-mock-info @@ -5,6 +5,8 @@ import unittest.mock def main() -> int: + assert sys.version_info[:2] == (3, 6), 'needs py36' + mock = __import__('mock') # avoid self-rewriting! # https://github.com/python/typeshed/pull/3923 @@ -24,7 +26,7 @@ def main() -> int: print( f'# GENERATED VIA {os.path.basename(sys.argv[0])}\n' f'# Using mock=={mock_version}\n' - f'MOCK_RENAMES = [\n{renames_s}\n]\n' + f'REPLACES[(3,)].update((\n{renames_s}\n))\n' f'# END GENERATED\n', ) diff --git a/testing/generate-python-future-info b/testing/generate-python-future-info new file mode 100755 index 0000000..d7398b0 --- /dev/null +++ b/testing/generate-python-future-info @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import builtins +import os.path +import sys + +import future.builtins + + +def main() -> int: + + names = sorted( + k for k, v in vars(future.builtins).items() + if getattr(builtins, k, None) is v + ) + print(f'# GENERATED VIA {os.path.basename(sys.argv[0])}') + print(f'# Using future=={future.__version__}') + print('REMOVALS[(3,)].update((') + print(" 'from builtins import *',") + for name in names: + print(f" 'from builtins import {name}',") + print('))') + print('# END GENERATED') + + return 0 + + +if __name__ == '__main__': + exit(main()) diff --git a/testing/generate-six-info b/testing/generate-six-info index b9f0baa..42071ea 100755 --- a/testing/generate-six-info +++ b/testing/generate-six-info @@ -48,8 +48,8 @@ def main() -> int: # https://github.com/python/typeshed/pull/3589 f'# GENERATED VIA {os.path.basename(sys.argv[0])}\n' # type: ignore f'# Using six=={six.__version__}\n' - f'SIX_REMOVALS = [\n{removes_s}]\n' - f'SIX_RENAMES = [\n{renames_s}]\n' + f'REMOVALS[(3,)].update((\n{removes_s}))\n' + f'REPLACES[(3,)].update((\n{renames_s}))\n' f'# END GENERATED\n', ) diff --git a/testing/generate-typing-rewrite-info b/testing/generate-typing-rewrite-info index c823eeb..e087132 100755 --- a/testing/generate-typing-rewrite-info +++ b/testing/generate-typing-rewrite-info @@ -17,37 +17,6 @@ else: from importlib_metadata import version -def _serialize_short_imports( - imports: Dict[Tuple[float, float], List[str]], -) -> str: - return '\n'.join( - f" ('py{v[0]}{v[1]}', {tuple(attrs)!r})," - for v, attrs in sorted(imports.items()) if attrs - ) - - -def _serialize_imports( - imports: Dict[Tuple[float, float], List[str]], -) -> str: - parts = [] - for v, attrs in sorted(imports.items()): - if not attrs: - continue - parts.append(' (') - parts.append(f" 'py{v[0]}{v[1]}', (") - accum = f' {attrs[0]!r},' - for symbol in attrs[1:]: - if len(accum) + len(repr(symbol)) + 2 < 80: - accum += f' {symbol!r},' - else: - parts.append(f'{accum}') - accum = f' {symbol!r},' - parts.append(f'{accum}') - parts.append(' ),') - parts.append(' ),') - return '\n'.join(parts) - - def main() -> int: flake8_typing_imports_version = version('flake8-typing-imports') mypy_extensions_version = version('mypy_extensions') @@ -75,41 +44,48 @@ def main() -> int: # --pyXX-plus assumes the min --pyXX so group symbols by their # rounded up major version - symbols_rounded_up: Dict[Tuple[float, float], Set[str]] = defaultdict(set) + symbols_rounded_up: Dict[Tuple[int, int], Set[str]] = defaultdict(set) for v, attrs in sorted(symbols.items()): symbols_rounded_up[v.major, v.minor + int(v.patch != 0)] |= attrs # combine 3.5 and 3.6 because this lib is 3.6.1+ symbols_rounded_up[(3, 6)] |= symbols_rounded_up.pop((3, 5)) - deltas: Dict[Tuple[float, float], Set[str]] = defaultdict(set) + deltas: Dict[Tuple[int, int], Set[str]] = defaultdict(set) prev: Set[str] = set() for v, attrs in sorted(symbols_rounded_up.items()): deltas[v] = attrs - prev prev = attrs - mypy_extensions_added: Dict[Tuple[float, float], List[str]] = {} - typing_extensions_added: Dict[Tuple[float, float], List[str]] = {} + mypy_extensions_added: Dict[Tuple[int, int], List[str]] = {} + typing_extensions_added: Dict[Tuple[int, int], List[str]] = {} for v, attrs in deltas.items(): mypy_extensions_added[v] = sorted(attrs & mypy_extensions_all) typing_extensions_added[v] = sorted(attrs & typing_extensions_all) - print( - f'# GENERATED VIA {os.path.basename(sys.argv[0])}\n' - f'# Using: \n' - f'# flake8-typing-imports=={flake8_typing_imports_version}\n' - f'# mypy_extensions=={mypy_extensions_version}\n' - f'# typing_extensions=={typing_extensions_version}\n' - f'MYPY_EXTENSIONS_IMPORTS: ' - f'Tuple[Tuple[str, Tuple[str, ...]], ...] = (\n' - f'{_serialize_short_imports(mypy_extensions_added)}\n' - f')\n' - f'TYPING_EXTENSIONS_IMPORTS: ' - f'Tuple[Tuple[str, Tuple[str, ...]], ...] = (\n' - f'{_serialize_imports(typing_extensions_added)}\n' - f')\n' - f'# END GENERATED\n', - ) + print(f'# GENERATED VIA {os.path.basename(sys.argv[0])}') + print('# Using:') + print(f'# flake8-typing-imports=={flake8_typing_imports_version}') + print(f'# mypy_extensions=={mypy_extensions_version}') + print(f'# typing_extensions=={typing_extensions_version}') + + for k, v in sorted(mypy_extensions_added.items()): + if not v: + continue + print(f'REPLACES[{k}].update((') + for symbol in sorted(v): + print(f" 'mypy_extensions=typing:{symbol}',") + print('))') + + for k, v in sorted(typing_extensions_added.items()): + if not v: + continue + print(f'REPLACES[{k}].update((') + for symbol in sorted(v): + print(f" 'typing_extensions=typing:{symbol}',") + print('))') + + print('# END GENERATED') return 0