From 9f0572d56203efd7c1848fe5ba46b649f5c5b8d7 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sat, 24 Oct 2020 18:05:11 -0700 Subject: [PATCH 1/2] tests / scripts: blacken and isort I often run black and isort in typeshed root and then have to undo these changes. --- scripts/migrate_script.py | 44 ++++++++---------- tests/check_consistent.py | 14 +++--- tests/mypy_selftest.py | 2 +- tests/mypy_test.py | 94 +++++++++++++++++++-------------------- tests/mypy_test_suite.py | 25 +++++------ tests/pytype_test.py | 20 ++++----- tests/stubtest_test.py | 2 +- tests/stubtest_unused.py | 8 ++-- 8 files changed, 96 insertions(+), 113 deletions(-) diff --git a/scripts/migrate_script.py b/scripts/migrate_script.py index 170d31ae48e7..45ecaef39533 100644 --- a/scripts/migrate_script.py +++ b/scripts/migrate_script.py @@ -7,9 +7,8 @@ import os import os.path import shutil - from dataclasses import dataclass -from typing import Optional, List, Set, Tuple +from typing import List, Optional, Set, Tuple # These names may be still discussed so I make them constants. STDLIB_NAMESPACE = "stdlib" @@ -62,6 +61,7 @@ # The latter two are distinguished by is_dir flag. class PackageBase: """Common attributes for packages/modules""" + path: str # full initial path like stdlib/2and3/argparse.pyi is_dir: bool @@ -79,6 +79,7 @@ def name(self) -> str: @dataclass class StdLibPackage(PackageBase): """Package/module in standard library.""" + path: str py_version: Optional[str] # Can be omitted for Python 2 only packages. is_dir: bool @@ -93,8 +94,7 @@ class ThirdPartyPackage(PackageBase): requires: List[str] # distributions this depends on -def add_stdlib_packages_from(subdir: str, packages: List[StdLibPackage], - py_version: Optional[str]) -> None: +def add_stdlib_packages_from(subdir: str, packages: List[StdLibPackage], py_version: Optional[str]) -> None: """Add standard library packages/modules from a given stdlib/xxx subdirectory. Append to packages list in-place, use py_version as the minimal supported version. @@ -120,30 +120,27 @@ def collect_stdlib_packages() -> Tuple[List[StdLibPackage], List[StdLibPackage]] return stdlib, py2_stdlib -def add_third_party_packages_from(subdir: str, packages: List[ThirdPartyPackage], - py2_compatible: bool, py3_compatible: bool) -> None: +def add_third_party_packages_from( + subdir: str, packages: List[ThirdPartyPackage], py2_compatible: bool, py3_compatible: bool +) -> None: """Add third party packages/modules from a given third_party/xxx subdirectory.""" for name in os.listdir(subdir): path = os.path.join(subdir, name) - packages.append(ThirdPartyPackage(path, py2_compatible, py3_compatible, - requires=[], is_dir=os.path.isdir(path))) + packages.append(ThirdPartyPackage(path, py2_compatible, py3_compatible, requires=[], is_dir=os.path.isdir(path))) def collect_third_party_packages() -> Tuple[List[ThirdPartyPackage], List[ThirdPartyPackage]]: """Collect third party packages/modules from all current third_party/xxx sub-directories.""" third_party: List[ThirdPartyPackage] = [] py2_third_party: List[ThirdPartyPackage] = [] - add_third_party_packages_from("third_party/3", third_party, - py2_compatible=False, py3_compatible=True) - add_third_party_packages_from("third_party/2and3", third_party, - py2_compatible=True, py3_compatible=True) + add_third_party_packages_from("third_party/3", third_party, py2_compatible=False, py3_compatible=True) + add_third_party_packages_from("third_party/2and3", third_party, py2_compatible=True, py3_compatible=True) # We special-case Python 2 for third party packages like six. subdir = "third_party/2" py3_packages = os.listdir("third_party/3") for name in os.listdir(subdir): path = os.path.join(subdir, name) - package = ThirdPartyPackage(path, py2_compatible=True, py3_compatible=False, - requires=[], is_dir=os.path.isdir(path)) + package = ThirdPartyPackage(path, py2_compatible=True, py3_compatible=False, requires=[], is_dir=os.path.isdir(path)) if name in py3_packages: # If there is a package with the same name in /2 and /3, we add the former to # a separate list, packages from there will be put into /python2 sub-directories. @@ -168,19 +165,19 @@ def get_top_imported_names(file: str) -> Set[str]: for node in ast.walk(parsed): if isinstance(node, ast.Import): for name in node.names: - top_imported.add(name.name.split('.')[0]) + top_imported.add(name.name.split(".")[0]) elif isinstance(node, ast.ImportFrom): if node.level > 0: # Relative imports always refer to the current package. continue assert node.module - top_imported.add(node.module.split('.')[0]) + top_imported.add(node.module.split(".")[0]) return top_imported -def populate_requirements(package: ThirdPartyPackage, - stdlib: List[str], py2_stdlib: List[str], - known_distributions: Set[str]) -> None: +def populate_requirements( + package: ThirdPartyPackage, stdlib: List[str], py2_stdlib: List[str], known_distributions: Set[str] +) -> None: """Generate requirements using imports found in a package.""" assert not package.requires, "Populate must be called once" if not package.is_dir: @@ -272,14 +269,12 @@ def generate_metadata(package: ThirdPartyPackage, py2_packages: List[str]) -> st if not package.py3_compatible: lines.append("python3 = false") if package.requires: - distributions = [f'"types-{package_to_distribution.get(dep, dep)}"' - for dep in package.requires] + distributions = [f'"types-{package_to_distribution.get(dep, dep)}"' for dep in package.requires] lines.append(f"requires = [{', '.join(distributions)}]") return "\n".join(lines) -def copy_third_party(packages: List[ThirdPartyPackage], - py2_packages: List[ThirdPartyPackage]) -> None: +def copy_third_party(packages: List[ThirdPartyPackage], py2_packages: List[ThirdPartyPackage]) -> None: """Refactor the third party part using collected metadata.""" third_party_dir = os.path.join(OUTPUT_DIR, THIRD_PARTY_NAMESPACE) os.makedirs(third_party_dir, exist_ok=True) @@ -325,8 +320,7 @@ def main() -> None: py2_stdlib_names += [package.name for package in stdlib if package.py_version == "2.7"] # Collect all known distributions (for sanity checks). - known_distributions = {package_to_distribution.get(package.name, package.name) - for package in third_party + py2_third_party} + known_distributions = {package_to_distribution.get(package.name, package.name) for package in third_party + py2_third_party} # Compute dependencies between third party packages/modules to populate metadata. for package in third_party + py2_third_party: diff --git a/tests/check_consistent.py b/tests/check_consistent.py index f5280b368daf..8abb09610a65 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -7,21 +7,21 @@ # manually update both files, and this test verifies that they are # identical. The list below indicates which sets of files must match. -import os import filecmp +import os consistent_files = [ - {'stdlib/2/builtins.pyi', 'stdlib/2/__builtin__.pyi'}, - {'stdlib/2and3/threading.pyi', 'stdlib/2and3/_dummy_threading.pyi'}, + {"stdlib/2/builtins.pyi", "stdlib/2/__builtin__.pyi"}, + {"stdlib/2and3/threading.pyi", "stdlib/2and3/_dummy_threading.pyi"}, ] def main(): - files = [os.path.join(root, file) for root, dir, files in os.walk('.') for file in files] - no_symlink = 'You cannot use symlinks in typeshed, please copy {} to its link.' + files = [os.path.join(root, file) for root, dir, files in os.walk(".") for file in files] + no_symlink = "You cannot use symlinks in typeshed, please copy {} to its link." for file in files: _, ext = os.path.splitext(file) - if ext == '.pyi' and os.path.islink(file): + if ext == ".pyi" and os.path.islink(file): raise ValueError(no_symlink.format(file)) for file1, *others in consistent_files: f1 = os.path.join(os.getcwd(), file1) @@ -34,5 +34,5 @@ def main(): ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tests/mypy_selftest.py b/tests/mypy_selftest.py index 3f891579fdb2..38efbf4bf4e4 100755 --- a/tests/mypy_selftest.py +++ b/tests/mypy_selftest.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 """Script to run mypy against its own code base.""" -from pathlib import Path import subprocess import sys import tempfile +from pathlib import Path MYPY_VERSION = "0.790" diff --git a/tests/mypy_test.py b/tests/mypy_test.py index f32da57ff2c3..a495d65e49e6 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -12,26 +12,26 @@ 5. Repeat steps 2-4 for other mypy runs (e.g. --py2) """ +import argparse import os import re import sys -import argparse -parser = argparse.ArgumentParser(description="Test runner for typeshed. " - "Patterns are unanchored regexps on the full path.") -parser.add_argument('-v', '--verbose', action='count', default=0, help="More output") -parser.add_argument('-n', '--dry-run', action='store_true', help="Don't actually run mypy") -parser.add_argument('-x', '--exclude', type=str, nargs='*', help="Exclude pattern") -parser.add_argument('-p', '--python-version', type=str, nargs='*', - help="These versions only (major[.minor])") -parser.add_argument('--platform', - help="Run mypy for a certain OS platform (defaults to sys.platform)") -parser.add_argument('--warn-unused-ignores', action='store_true', - help="Run mypy with --warn-unused-ignores " - "(hint: only get rid of warnings that are " - "unused for all platforms and Python versions)") +parser = argparse.ArgumentParser(description="Test runner for typeshed. " "Patterns are unanchored regexps on the full path.") +parser.add_argument("-v", "--verbose", action="count", default=0, help="More output") +parser.add_argument("-n", "--dry-run", action="store_true", help="Don't actually run mypy") +parser.add_argument("-x", "--exclude", type=str, nargs="*", help="Exclude pattern") +parser.add_argument("-p", "--python-version", type=str, nargs="*", help="These versions only (major[.minor])") +parser.add_argument("--platform", help="Run mypy for a certain OS platform (defaults to sys.platform)") +parser.add_argument( + "--warn-unused-ignores", + action="store_true", + help="Run mypy with --warn-unused-ignores " + "(hint: only get rid of warnings that are " + "unused for all platforms and Python versions)", +) -parser.add_argument('filter', type=str, nargs='*', help="Include pattern (default all)") +parser.add_argument("filter", type=str, nargs="*", help="Include pattern (default all)") def log(args, *varargs): @@ -41,36 +41,35 @@ def log(args, *varargs): def match(fn, args, exclude_list): if exclude_list.match(fn): - log(args, fn, 'exluded by exclude list') + log(args, fn, "exluded by exclude list") return False if not args.filter and not args.exclude: - log(args, fn, 'accept by default') + log(args, fn, "accept by default") return True if args.exclude: for f in args.exclude: if re.search(f, fn): - log(args, fn, 'excluded by pattern', f) + log(args, fn, "excluded by pattern", f) return False if args.filter: for f in args.filter: if re.search(f, fn): - log(args, fn, 'accepted by pattern', f) + log(args, fn, "accepted by pattern", f) return True if args.filter: - log(args, fn, 'rejected (no pattern matches)') + log(args, fn, "rejected (no pattern matches)") return False - log(args, fn, 'accepted (no exclude pattern matches)') + log(args, fn, "accepted (no exclude pattern matches)") return True def libpath(major, minor): - versions = ['%d.%d' % (major, minor) - for minor in reversed(range(minor + 1))] + versions = ["%d.%d" % (major, minor) for minor in reversed(range(minor + 1))] versions.append(str(major)) - versions.append('2and3') + versions.append("2and3") paths = [] for v in versions: - for top in ['stdlib', 'third_party']: + for top in ["stdlib", "third_party"]: p = os.path.join(top, v) if os.path.isdir(p): paths.append(p) @@ -81,8 +80,7 @@ def main(): args = parser.parse_args() with open(os.path.join(os.path.dirname(__file__), "mypy_exclude_list.txt")) as f: - exclude_list = re.compile("(%s)$" % "|".join( - re.findall(r"^\s*([^\s#]+)\s*(?:#.*)?$", f.read(), flags=re.M))) + exclude_list = re.compile("(%s)$" % "|".join(re.findall(r"^\s*([^\s#]+)\s*(?:#.*)?$", f.read(), flags=re.M))) try: from mypy.main import main as mypy_main @@ -92,8 +90,7 @@ def main(): versions = [(3, 9), (3, 8), (3, 7), (3, 6), (3, 5), (2, 7)] if args.python_version: - versions = [v for v in versions - if any(('%d.%d' % v).startswith(av) for av in args.python_version)] + versions = [v for v in versions if any(("%d.%d" % v).startswith(av) for av in args.python_version)] if not versions: print("--- no versions selected ---") sys.exit(1) @@ -103,51 +100,50 @@ def main(): for major, minor in versions: roots = libpath(major, minor) files = [] - seen = {'__builtin__', 'builtins', 'typing'} # Always ignore these. + seen = {"__builtin__", "builtins", "typing"} # Always ignore these. for root in roots: names = os.listdir(root) for name in names: full = os.path.join(root, name) mod, ext = os.path.splitext(name) - if mod in seen or mod.startswith('.'): + if mod in seen or mod.startswith("."): continue - if ext in ['.pyi', '.py']: + if ext in [".pyi", ".py"]: if match(full, args, exclude_list): seen.add(mod) files.append(full) - elif (os.path.isfile(os.path.join(full, '__init__.pyi')) or - os.path.isfile(os.path.join(full, '__init__.py'))): + elif os.path.isfile(os.path.join(full, "__init__.pyi")) or os.path.isfile(os.path.join(full, "__init__.py")): for r, ds, fs in os.walk(full): ds.sort() fs.sort() for f in fs: m, x = os.path.splitext(f) - if x in ['.pyi', '.py']: + if x in [".pyi", ".py"]: fn = os.path.join(r, f) if match(fn, args, exclude_list): seen.add(mod) files.append(fn) if files: runs += 1 - flags = ['--python-version', '%d.%d' % (major, minor)] - flags.append('--strict-optional') - flags.append('--no-site-packages') - flags.append('--show-traceback') - flags.append('--no-implicit-optional') - flags.append('--disallow-any-generics') - flags.append('--disallow-subclassing-any') + flags = ["--python-version", "%d.%d" % (major, minor)] + flags.append("--strict-optional") + flags.append("--no-site-packages") + flags.append("--show-traceback") + flags.append("--no-implicit-optional") + flags.append("--disallow-any-generics") + flags.append("--disallow-subclassing-any") if args.warn_unused_ignores: - flags.append('--warn-unused-ignores') + flags.append("--warn-unused-ignores") if args.platform: - flags.extend(['--platform', args.platform]) - sys.argv = ['mypy'] + flags + files + flags.extend(["--platform", args.platform]) + sys.argv = ["mypy"] + flags + files if args.verbose: - print("running", ' '.join(sys.argv)) + print("running", " ".join(sys.argv)) else: - print("running mypy", ' '.join(flags), "# with", len(files), "files") + print("running mypy", " ".join(flags), "# with", len(files), "files") try: if not args.dry_run: - mypy_main('', sys.stdout, sys.stderr) + mypy_main("", sys.stdout, sys.stderr) except SystemExit as err: code = max(code, err.code) if code: @@ -158,5 +154,5 @@ def main(): sys.exit(1) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tests/mypy_test_suite.py b/tests/mypy_test_suite.py index fd7f23b105bb..f69ebdbf827d 100755 --- a/tests/mypy_test_suite.py +++ b/tests/mypy_test_suite.py @@ -1,28 +1,27 @@ #!/usr/bin/env python3 """Script to run mypy's test suite against this version of typeshed.""" -from pathlib import Path import shutil import subprocess import sys import tempfile +from pathlib import Path - -if __name__ == '__main__': +if __name__ == "__main__": with tempfile.TemporaryDirectory() as tempdir: dirpath = Path(tempdir) - subprocess.run(['python2.7', '-m', 'pip', 'install', '--user', 'typing'], check=True) - subprocess.run(['git', 'clone', '--depth', '1', 'git://github.com/python/mypy', - str(dirpath / 'mypy')], check=True) - subprocess.run([sys.executable, '-m', 'pip', 'install', '-U', '-r', - str(dirpath / 'mypy/test-requirements.txt')], check=True) - shutil.copytree('stdlib', str(dirpath / 'mypy/mypy/typeshed/stdlib')) - shutil.copytree('third_party', str(dirpath / 'mypy/mypy/typeshed/third_party')) + subprocess.run(["python2.7", "-m", "pip", "install", "--user", "typing"], check=True) + subprocess.run(["git", "clone", "--depth", "1", "git://github.com/python/mypy", str(dirpath / "mypy")], check=True) + subprocess.run( + [sys.executable, "-m", "pip", "install", "-U", "-r", str(dirpath / "mypy/test-requirements.txt")], check=True + ) + shutil.copytree("stdlib", str(dirpath / "mypy/mypy/typeshed/stdlib")) + shutil.copytree("third_party", str(dirpath / "mypy/mypy/typeshed/third_party")) try: - subprocess.run(['pytest', '-n12'], cwd=str(dirpath / 'mypy'), check=True) + subprocess.run(["pytest", "-n12"], cwd=str(dirpath / "mypy"), check=True) except subprocess.CalledProcessError as e: - print('mypy tests failed', file=sys.stderr) + print("mypy tests failed", file=sys.stderr) sys.exit(e.returncode) else: - print('mypy tests succeeded', file=sys.stderr) + print("mypy tests succeeded", file=sys.stderr) sys.exit(0) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index acc235f8eacd..bd8c6bf91f59 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -49,7 +49,11 @@ def create_parser() -> argparse.ArgumentParser: "--print-stderr", action="store_true", default=False, help="Print stderr every time an error is encountered." ) parser.add_argument( - "files", metavar="FILE", type=str, nargs="*", help="Files or directories to check. (Default: Check all files.)", + "files", + metavar="FILE", + type=str, + nargs="*", + help="Files or directories to check. (Default: Check all files.)", ) return parser @@ -82,10 +86,8 @@ def load_exclude_list(typeshed_location: str) -> List[str]: def run_pytype(*, filename: str, python_version: str, typeshed_location: str) -> Optional[str]: """Runs pytype, returning the stderr if any.""" options = pytype_config.Options.create( - filename, - module_name=_get_module_name(filename), - parse_pyi=True, - python_version=python_version) + filename, module_name=_get_module_name(filename), parse_pyi=True, python_version=python_version + ) old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET) os.environ[TYPESHED_HOME] = typeshed_location try: @@ -162,13 +164,7 @@ def find_stubs_in_paths(paths: Sequence[str]) -> List[str]: return filenames -def run_all_tests( - *, - files_to_test: Sequence[Tuple[str, int]], - typeshed_location: str, - print_stderr: bool, - dry_run: bool -) -> None: +def run_all_tests(*, files_to_test: Sequence[Tuple[str, int]], typeshed_location: str, print_stderr: bool, dry_run: bool) -> None: bad = [] errors = 0 total_tests = len(files_to_test) diff --git a/tests/stubtest_test.py b/tests/stubtest_test.py index 31605e1dfcc9..54a4fdaa8145 100755 --- a/tests/stubtest_test.py +++ b/tests/stubtest_test.py @@ -10,9 +10,9 @@ """ -from pathlib import Path import subprocess import sys +from pathlib import Path def run_stubtest(typeshed_dir: Path) -> int: diff --git a/tests/stubtest_unused.py b/tests/stubtest_unused.py index f960a9d45d7e..4221ceec2fd4 100755 --- a/tests/stubtest_unused.py +++ b/tests/stubtest_unused.py @@ -2,10 +2,10 @@ # Runs stubtest and prints each unused whitelist entry with filename. -from typing import List, Tuple import os.path import subprocess import sys +from typing import List, Tuple _UNUSED_NOTE = "note: unused whitelist entry " _WHITELIST_PATH = os.path.join("tests", "stubtest_whitelists") @@ -22,11 +22,9 @@ def main() -> int: def run_stubtest() -> List[str]: - proc = subprocess.run( - ["./tests/stubtest_test.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) + proc = subprocess.run(["./tests/stubtest_test.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = proc.stdout.decode("utf-8").splitlines() - return [line[len(_UNUSED_NOTE):].strip() for line in output if line.startswith(_UNUSED_NOTE)] + return [line[len(_UNUSED_NOTE) :].strip() for line in output if line.startswith(_UNUSED_NOTE)] def unused_files(unused: str) -> List[Tuple[str, str]]: From 06f1b51643a1ea022cf861dc4568d7dccfe5ca24 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sat, 24 Oct 2020 18:19:58 -0700 Subject: [PATCH 2/2] flake8: configure for black --- .flake8 | 1 + 1 file changed, 1 insertion(+) diff --git a/.flake8 b/.flake8 index 68e39f79be28..92efbe9072dc 100644 --- a/.flake8 +++ b/.flake8 @@ -14,6 +14,7 @@ [flake8] per-file-ignores = + *.py: E203, W503 *.pyi: E301, E302, E305, E501, E701, E741, F401, F403, F405, F822 # Since typing.pyi defines "overload" this is not recognized by flake8 as typing.overload. # Unfortunately, flake8 does not allow to "noqa" just a specific error inside the file itself.