From f18466662c7501d1ac7eab04dd28a6c6f30ff405 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Thu, 28 Jan 2021 23:29:13 -0800 Subject: [PATCH 01/22] Add --ignore-path Resolves #4675, resolves #9981 Additionally, we always ignore site-packages and node_modules. Also note that this doesn't really affect import discovery; it only directly affects passing files or packages to mypy. The additional check before suggesting "are you missing an __init__.py" didn't make any sense to me, so I removed it, appended to the message and downgraded the severity to note. --- docs/source/command_line.rst | 7 +++++++ docs/source/config_file.rst | 9 +++++++++ docs/source/running_mypy.rst | 3 ++- mypy/build.py | 14 ++++++-------- mypy/config_parser.py | 1 + mypy/find_sources.py | 9 ++++++++- mypy/main.py | 7 +++++++ mypy/modulefinder.py | 10 +++++++++- mypy/options.py | 2 ++ mypy/test/test_find_sources.py | 31 +++++++++++++++++++++++++++++++ mypy_self_check.ini | 1 + test-data/unit/cmdline.test | 16 ++-------------- 12 files changed, 85 insertions(+), 25 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 8b8e7e6ee928..e59027ac4192 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -49,6 +49,13 @@ for full details, see :ref:`running-mypy`. Asks mypy to type check the provided string as a program. +.. option:: --ignore-path + + Asks mypy to ignore a given file name, directory name or subpath while + recursively discovering files to check. This flag may be repeated multiple + times. + + Optional arguments ****************** diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index b747b31136cd..3c293eab7d91 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -197,6 +197,15 @@ section of the command line docs. This option may only be set in the global section (``[mypy]``). +.. confval:: ignore_path + + :type: comma-separated list of strings + + A comma-separated list of file names, directory names or subpaths which mypy + should ignore while recursively discovering files to check. + + This option may only be set in the global section (``[mypy]``). + .. confval:: namespace_packages :type: boolean diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst index 3498aaf07275..e2c3230a03f7 100644 --- a/docs/source/running_mypy.rst +++ b/docs/source/running_mypy.rst @@ -390,7 +390,8 @@ to modules to type check. - Mypy will check all paths provided that correspond to files. - Mypy will recursively discover and check all files ending in ``.py`` or - ``.pyi`` in directory paths provided. + ``.pyi`` in directory paths provided, after accounting for + :option:`--ignore-path `. - For each file to be checked, mypy will attempt to associate the file (e.g. ``project/foo/bar/baz.py``) with a fully qualified module name (e.g. diff --git a/mypy/build.py b/mypy/build.py index 0ea4f643d20b..ae4653828bc0 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2769,14 +2769,12 @@ def load_graph(sources: List[BuildSource], manager: BuildManager, "Duplicate module named '%s' (also at '%s')" % (st.id, graph[st.id].xpath), blocker=True, ) - p1 = len(pathlib.PurePath(st.xpath).parents) - p2 = len(pathlib.PurePath(graph[st.id].xpath).parents) - - if p1 != p2: - manager.errors.report( - -1, -1, - "Are you missing an __init__.py?" - ) + manager.errors.report( + -1, -1, + "Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid " + "checking one of them.", + severity='note' + ) manager.errors.raise_error() graph[st.id] = st diff --git a/mypy/config_parser.py b/mypy/config_parser.py index dd79869030e5..3e7bec4f64fa 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -83,6 +83,7 @@ def check_follow_imports(choice: str) -> str: 'custom_typing_module': str, 'custom_typeshed_dir': expand_path, 'mypy_path': lambda s: [expand_path(p.strip()) for p in re.split('[,:]', s)], + 'ignore_path': lambda s: [expand_path(p.strip()) for p in s.split(',')], 'files': split_and_match_files, 'quickstart_file': expand_path, 'junit_xml': expand_path, diff --git a/mypy/find_sources.py b/mypy/find_sources.py index 47d686cddcbc..7907c25ef437 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -91,6 +91,7 @@ def __init__(self, fscache: FileSystemCache, options: Options) -> None: self.fscache = fscache self.explicit_package_bases = get_explicit_package_bases(options) self.namespace_packages = options.namespace_packages + self.ignore_path = options.ignore_path def is_explicit_package_base(self, path: str) -> bool: assert self.explicit_package_bases @@ -103,9 +104,15 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]: names = sorted(self.fscache.listdir(path), key=keyfunc) for name in names: # Skip certain names altogether - if name == '__pycache__' or name.startswith('.') or name.endswith('~'): + if ( + name in ("__pycache__", "site-packages", "node_modules") + or name.startswith(".") + or name.endswith("~") + ): continue subpath = os.path.join(path, name) + if any(subpath.endswith(pattern.rstrip("/")) for pattern in self.ignore_path): + continue if self.fscache.isdir(subpath): sub_sources = self.find_sources_in_dir(subpath) diff --git a/mypy/main.py b/mypy/main.py index be2fab1bb0f9..d6e226ba968e 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -808,6 +808,13 @@ def add_invertible_flag(flag: str, code_group.add_argument( '--explicit-package-bases', action='store_true', help="Use current directory and MYPYPATH to determine module names of files passed") + code_group.add_argument( + "--ignore-path", + metavar="PATH", + action="append", + default=[], + help="File names, directory names or subpaths to avoid checking", + ) code_group.add_argument( '-m', '--module', action='append', metavar='MODULE', default=[], diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index d223058c0367..34033ae2fa9f 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -443,9 +443,17 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: names = sorted(self.fscache.listdir(package_path)) for name in names: # Skip certain names altogether - if name == '__pycache__' or name.startswith('.') or name.endswith('~'): + if ( + name in ("__pycache__", "site-packages", "node_modules") + or name.startswith(".") + or name.endswith("~") + ): continue subpath = os.path.join(package_path, name) + if self.options and any( + subpath.endswith(pattern.rstrip("/")) for pattern in self.options.ignore_path + ): + continue if self.fscache.isdir(subpath): # Only recurse into packages diff --git a/mypy/options.py b/mypy/options.py index 2a0b3e111442..953335f8536e 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -97,6 +97,8 @@ def __init__(self) -> None: # sufficient to determine module names for files. As a possible alternative, add a single # top-level __init__.py to your packages. self.explicit_package_bases = False + # File names, directory names or subpaths to avoid checking + self.ignore_path = [] # type: List[str] # disallow_any options self.disallow_any_generics = False diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 5cedec338bbc..2cc47cd4eef1 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -252,3 +252,34 @@ def test_find_sources_namespace_multi_dir(self) -> None: finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options) assert find_sources(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")] + + def test_find_sources_ignore_path(self) -> None: + options = Options() + options.namespace_packages = True + + finder = SourceFinder(FakeFSCache({"/dir/a.py", "/dir/venv/site-packages/b.py"}), options) + assert find_sources(finder, "/") == [("a", "/dir")] + + files = { + "/pkg/a1/b/c/d/e.py", + "/pkg/a1/b/f.py", + "/pkg/a2/__init__.py", + "/pkg/a2/b/c/d/e.py", + "/pkg/a2/b/f.py", + } + + options.ignore_path = ["/pkg/a1"] + finder = SourceFinder(FakeFSCache(files), options) + assert find_sources(finder, "/") == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("a2.b.f", "/pkg"), + ] + + options.ignore_path = ["f.py"] + finder = SourceFinder(FakeFSCache(files), options) + assert find_sources(finder, "/") == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("e", "/pkg/a1/b/c/d"), + ] diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 2b7ed2b157c5..15fea22c7dc2 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -19,3 +19,4 @@ pretty = True always_false = MYPYC plugins = misc/proper_plugin.py python_version = 3.5 +ignore_path = mypy/typeshed diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index c4d20ee78d61..87c3bd42b61f 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -59,7 +59,7 @@ undef undef [out] dir/a.py: error: Duplicate module named 'a' (also at 'dir/subdir/a.py') -dir/a.py: error: Are you missing an __init__.py? +dir/a.py: note: Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid checking one of them. == Return code: 2 [case testCmdlineNonPackageSlash] @@ -125,19 +125,7 @@ mypy: can't decode file 'a.py': unknown encoding: uft-8 # type: ignore [out] two/mod/__init__.py: error: Duplicate module named 'mod' (also at 'one/mod/__init__.py') -== Return code: 2 - -[case promptsForgotInit] -# cmd: mypy a.py one/mod/a.py -[file one/__init__.py] -# type: ignore -[file a.py] -# type: ignore -[file one/mod/a.py] -#type: ignore -[out] -one/mod/a.py: error: Duplicate module named 'a' (also at 'a.py') -one/mod/a.py: error: Are you missing an __init__.py? +two/mod/__init__.py: note: Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid checking one of them. == Return code: 2 [case testFlagsFile] From 75f09c2298cc49a0e721e9b12e6171c9853e1715 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 29 Jan 2021 00:19:40 -0800 Subject: [PATCH 02/22] fix flake8 --- mypy/build.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index ae4653828bc0..2f074cd6bacc 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -15,7 +15,6 @@ import gc import json import os -import pathlib import re import stat import sys @@ -2771,8 +2770,8 @@ def load_graph(sources: List[BuildSource], manager: BuildManager, ) manager.errors.report( -1, -1, - "Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid " - "checking one of them.", + "Are you missing an __init__.py? Alternatively, consider using --ignore-path to " + "avoid checking one of them.", severity='note' ) From ede5ac80b60c9fbcb5dc3ea4cb3844e73dcdf56b Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 29 Jan 2021 12:02:12 -0800 Subject: [PATCH 03/22] fix logic --- mypy/config_parser.py | 2 +- mypy/find_sources.py | 4 ++-- mypy/main.py | 2 ++ mypy/modulefinder.py | 11 ++++++++++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/mypy/config_parser.py b/mypy/config_parser.py index 3e7bec4f64fa..a2f9d6d6c18e 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -83,7 +83,7 @@ def check_follow_imports(choice: str) -> str: 'custom_typing_module': str, 'custom_typeshed_dir': expand_path, 'mypy_path': lambda s: [expand_path(p.strip()) for p in re.split('[,:]', s)], - 'ignore_path': lambda s: [expand_path(p.strip()) for p in s.split(',')], + 'ignore_path': lambda s: [expand_path(p.strip()).replace("/", os.sep) for p in s.split(",")], 'files': split_and_match_files, 'quickstart_file': expand_path, 'junit_xml': expand_path, diff --git a/mypy/find_sources.py b/mypy/find_sources.py index 7907c25ef437..3c29dc3864b6 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -6,7 +6,7 @@ from typing import List, Sequence, Set, Tuple, Optional from typing_extensions import Final -from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path +from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path, matches_ignore_pattern from mypy.fscache import FileSystemCache from mypy.options import Options @@ -111,7 +111,7 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]: ): continue subpath = os.path.join(path, name) - if any(subpath.endswith(pattern.rstrip("/")) for pattern in self.ignore_path): + if any(matches_ignore_pattern(subpath, pattern) for pattern in self.ignore_path): continue if self.fscache.isdir(subpath): diff --git a/mypy/main.py b/mypy/main.py index d6e226ba968e..dc9eea486ec9 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -954,6 +954,8 @@ def set_strict_flags() -> None: if options.logical_deps: options.cache_fine_grained = True + options.ignore_path = [p.replace("/", os.sep) for p in options.ignore_path] + # Set target. if special_opts.modules + special_opts.packages: options.build_type = BuildType.MODULE diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 34033ae2fa9f..4ba4242fa076 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -451,7 +451,7 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: continue subpath = os.path.join(package_path, name) if self.options and any( - subpath.endswith(pattern.rstrip("/")) for pattern in self.options.ignore_path + matches_ignore_pattern(subpath, pattern) for pattern in self.options.ignore_path ): continue @@ -475,6 +475,15 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: return sources +def matches_ignore_pattern(path: str, pattern: str) -> bool: + path_components = path.split(os.sep) + pattern_components = pattern.split(os.sep) + return all( + path == pattern + for path, pattern in zip(reversed(path_components), reversed(pattern_components)) + ) + + def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> bool: """Check that all packages containing id have a __init__ file.""" if path.endswith(('__init__.py', '__init__.pyi')): From f3ff83fb3babff253016562fee85398be7e36a6b Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 29 Jan 2021 15:34:26 -0800 Subject: [PATCH 04/22] fix windows, add another test --- mypy/modulefinder.py | 1 + mypy/test/test_find_sources.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 4ba4242fa076..f23e52f51fbc 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -476,6 +476,7 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: def matches_ignore_pattern(path: str, pattern: str) -> bool: + path = os.path.splitdrive(path)[1] path_components = path.split(os.sep) pattern_components = pattern.split(os.sep) return all( diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 2cc47cd4eef1..71ee1df38b1c 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -268,6 +268,10 @@ def test_find_sources_ignore_path(self) -> None: "/pkg/a2/b/f.py", } + options.ignore_path = ["/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b"] + finder = SourceFinder(FakeFSCache(files), options) + assert len(find_sources(finder, "/")) == len(files) + options.ignore_path = ["/pkg/a1"] finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ @@ -283,3 +287,17 @@ def test_find_sources_ignore_path(self) -> None: ("a2.b.c.d.e", "/pkg"), ("e", "/pkg/a1/b/c/d"), ] + + files = { + "pkg/a1/b/c/d/e.py", + "pkg/a1/b/f.py", + "pkg/a2/__init__.py", + "pkg/a2/b/c/d/e.py", + "pkg/a2/b/f.py", + } + + options.ignore_path = [ + "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/pkg/a1", "/pkg/a2" + ] + finder = SourceFinder(FakeFSCache(files), options) + assert len(find_sources(finder, "/")) == len(files) From f942ff7b162c79f872a4281074ae22d4144b034d Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 29 Jan 2021 15:48:16 -0800 Subject: [PATCH 05/22] add a check and another test This shouldn't actually come up in practice, since we use absolute paths pretty aggressively. --- mypy/modulefinder.py | 2 ++ mypy/test/test_find_sources.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index f23e52f51fbc..a96afd5153df 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -479,6 +479,8 @@ def matches_ignore_pattern(path: str, pattern: str) -> bool: path = os.path.splitdrive(path)[1] path_components = path.split(os.sep) pattern_components = pattern.split(os.sep) + if len(path_components) < len(pattern_components): + return False return all( path == pattern for path, pattern in zip(reversed(path_components), reversed(pattern_components)) diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 71ee1df38b1c..37d713a5a815 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -268,7 +268,10 @@ def test_find_sources_ignore_path(self) -> None: "/pkg/a2/b/f.py", } - options.ignore_path = ["/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b"] + options.ignore_path = [ + "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py" + "xxx/pkg/a2/b/f.py" + ] finder = SourceFinder(FakeFSCache(files), options) assert len(find_sources(finder, "/")) == len(files) @@ -297,7 +300,8 @@ def test_find_sources_ignore_path(self) -> None: } options.ignore_path = [ - "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/pkg/a1", "/pkg/a2" + "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py", + "xxx/pkg/a2/b/f.py", "/pkg/a1", "/pkg/a2" ] finder = SourceFinder(FakeFSCache(files), options) assert len(find_sources(finder, "/")) == len(files) From 188b10f4bd93625d3b24b11a542c4387910bbc08 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 29 Jan 2021 16:08:19 -0800 Subject: [PATCH 06/22] hopefully actually fix windows --- mypy/config_parser.py | 2 +- mypy/main.py | 2 -- mypy/modulefinder.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mypy/config_parser.py b/mypy/config_parser.py index a2f9d6d6c18e..ac5cf257740f 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -83,7 +83,7 @@ def check_follow_imports(choice: str) -> str: 'custom_typing_module': str, 'custom_typeshed_dir': expand_path, 'mypy_path': lambda s: [expand_path(p.strip()) for p in re.split('[,:]', s)], - 'ignore_path': lambda s: [expand_path(p.strip()).replace("/", os.sep) for p in s.split(",")], + 'ignore_path': lambda s: [expand_path(p.strip()) for p in s.split(",")], 'files': split_and_match_files, 'quickstart_file': expand_path, 'junit_xml': expand_path, diff --git a/mypy/main.py b/mypy/main.py index dc9eea486ec9..d6e226ba968e 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -954,8 +954,6 @@ def set_strict_flags() -> None: if options.logical_deps: options.cache_fine_grained = True - options.ignore_path = [p.replace("/", os.sep) for p in options.ignore_path] - # Set target. if special_opts.modules + special_opts.packages: options.build_type = BuildType.MODULE diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index a96afd5153df..b2e6d1880289 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -478,7 +478,7 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: def matches_ignore_pattern(path: str, pattern: str) -> bool: path = os.path.splitdrive(path)[1] path_components = path.split(os.sep) - pattern_components = pattern.split(os.sep) + pattern_components = pattern.replace(os.sep, "/").split("/") if len(path_components) < len(pattern_components): return False return all( From 18bdd3b2170169226d1a985df77c86c8a85106c6 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 29 Jan 2021 16:13:23 -0800 Subject: [PATCH 07/22] more test! --- mypy/test/test_find_sources.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 37d713a5a815..ac196c1078a0 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -257,6 +257,7 @@ def test_find_sources_ignore_path(self) -> None: options = Options() options.namespace_packages = True + # special cased name finder = SourceFinder(FakeFSCache({"/dir/a.py", "/dir/venv/site-packages/b.py"}), options) assert find_sources(finder, "/") == [("a", "/dir")] @@ -268,13 +269,7 @@ def test_find_sources_ignore_path(self) -> None: "/pkg/a2/b/f.py", } - options.ignore_path = [ - "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py" - "xxx/pkg/a2/b/f.py" - ] - finder = SourceFinder(FakeFSCache(files), options) - assert len(find_sources(finder, "/")) == len(files) - + # directory name options.ignore_path = ["/pkg/a1"] finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ @@ -283,6 +278,7 @@ def test_find_sources_ignore_path(self) -> None: ("a2.b.f", "/pkg"), ] + # file name options.ignore_path = ["f.py"] finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ @@ -291,6 +287,24 @@ def test_find_sources_ignore_path(self) -> None: ("e", "/pkg/a1/b/c/d"), ] + # subpath + options.ignore_path = ["b/c"] + finder = SourceFinder(FakeFSCache(files), options) + assert find_sources(finder, "/") == [ + ("a2", "/pkg"), + ("a2.b.f", "/pkg"), + ("f", "/pkg/a1/b"), + ] + + # nothing should be ignored as a result of this + options.ignore_path = [ + "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py" + "xxx/pkg/a2/b/f.py" + ] + finder = SourceFinder(FakeFSCache(files), options) + assert len(find_sources(finder, "/")) == len(files) + + # nothing should be ignored as a result of this files = { "pkg/a1/b/c/d/e.py", "pkg/a1/b/f.py", @@ -298,7 +312,6 @@ def test_find_sources_ignore_path(self) -> None: "pkg/a2/b/c/d/e.py", "pkg/a2/b/f.py", } - options.ignore_path = [ "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py", "xxx/pkg/a2/b/f.py", "/pkg/a1", "/pkg/a2" From ea04b251773ed8ca88d386ecc3d3f4453a2e47c3 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 29 Jan 2021 16:50:28 -0800 Subject: [PATCH 08/22] rearrange tests --- mypy/test/test_find_sources.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index ac196c1078a0..c8532af55738 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -269,8 +269,17 @@ def test_find_sources_ignore_path(self) -> None: "/pkg/a2/b/f.py", } + # file name + options.ignore_path = ["f.py"] + finder = SourceFinder(FakeFSCache(files), options) + assert find_sources(finder, "/") == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("e", "/pkg/a1/b/c/d"), + ] + # directory name - options.ignore_path = ["/pkg/a1"] + options.ignore_path = ["a1"] finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), @@ -278,16 +287,15 @@ def test_find_sources_ignore_path(self) -> None: ("a2.b.f", "/pkg"), ] - # file name - options.ignore_path = ["f.py"] + # paths + options.ignore_path = ["/pkg/a1"] finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), ("a2.b.c.d.e", "/pkg"), - ("e", "/pkg/a1/b/c/d"), + ("a2.b.f", "/pkg"), ] - # subpath options.ignore_path = ["b/c"] finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ From 32c7f7c432910e37f1cc41d9b16c2b409cfd86c0 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Sun, 31 Jan 2021 16:14:29 -0800 Subject: [PATCH 09/22] rename --ignore-path to --exclude --- docs/source/command_line.rst | 4 ++-- docs/source/config_file.rst | 2 +- docs/source/running_mypy.rst | 2 +- mypy/build.py | 2 +- mypy/config_parser.py | 2 +- mypy/find_sources.py | 6 +++--- mypy/main.py | 2 +- mypy/modulefinder.py | 4 ++-- mypy/options.py | 2 +- mypy/test/test_find_sources.py | 14 +++++++------- mypy_self_check.ini | 2 +- test-data/unit/cmdline.test | 4 ++-- 12 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index e59027ac4192..6747ff46269e 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -49,9 +49,9 @@ for full details, see :ref:`running-mypy`. Asks mypy to type check the provided string as a program. -.. option:: --ignore-path +.. option:: --exclude - Asks mypy to ignore a given file name, directory name or subpath while + Asks mypy to exclude a given file name, directory name or subpath while recursively discovering files to check. This flag may be repeated multiple times. diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index 3c293eab7d91..d7562531e74c 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -197,7 +197,7 @@ section of the command line docs. This option may only be set in the global section (``[mypy]``). -.. confval:: ignore_path +.. confval:: exclude :type: comma-separated list of strings diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst index e2c3230a03f7..ad55d3f162f1 100644 --- a/docs/source/running_mypy.rst +++ b/docs/source/running_mypy.rst @@ -391,7 +391,7 @@ to modules to type check. - Mypy will recursively discover and check all files ending in ``.py`` or ``.pyi`` in directory paths provided, after accounting for - :option:`--ignore-path `. + :option:`--exclude `. - For each file to be checked, mypy will attempt to associate the file (e.g. ``project/foo/bar/baz.py``) with a fully qualified module name (e.g. diff --git a/mypy/build.py b/mypy/build.py index 2f074cd6bacc..1aad548a6e8c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2770,7 +2770,7 @@ def load_graph(sources: List[BuildSource], manager: BuildManager, ) manager.errors.report( -1, -1, - "Are you missing an __init__.py? Alternatively, consider using --ignore-path to " + "Are you missing an __init__.py? Alternatively, consider using --exclude to " "avoid checking one of them.", severity='note' ) diff --git a/mypy/config_parser.py b/mypy/config_parser.py index ac5cf257740f..5d116ace41d1 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -83,7 +83,7 @@ def check_follow_imports(choice: str) -> str: 'custom_typing_module': str, 'custom_typeshed_dir': expand_path, 'mypy_path': lambda s: [expand_path(p.strip()) for p in re.split('[,:]', s)], - 'ignore_path': lambda s: [expand_path(p.strip()) for p in s.split(",")], + 'exclude': lambda s: [expand_path(p.strip()) for p in s.split(",")], 'files': split_and_match_files, 'quickstart_file': expand_path, 'junit_xml': expand_path, diff --git a/mypy/find_sources.py b/mypy/find_sources.py index 3c29dc3864b6..4adaf31e8915 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -6,7 +6,7 @@ from typing import List, Sequence, Set, Tuple, Optional from typing_extensions import Final -from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path, matches_ignore_pattern +from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path, matches_exclude_pattern from mypy.fscache import FileSystemCache from mypy.options import Options @@ -91,7 +91,7 @@ def __init__(self, fscache: FileSystemCache, options: Options) -> None: self.fscache = fscache self.explicit_package_bases = get_explicit_package_bases(options) self.namespace_packages = options.namespace_packages - self.ignore_path = options.ignore_path + self.exclude = options.exclude def is_explicit_package_base(self, path: str) -> bool: assert self.explicit_package_bases @@ -111,7 +111,7 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]: ): continue subpath = os.path.join(path, name) - if any(matches_ignore_pattern(subpath, pattern) for pattern in self.ignore_path): + if any(matches_exclude_pattern(subpath, pattern) for pattern in self.exclude): continue if self.fscache.isdir(subpath): diff --git a/mypy/main.py b/mypy/main.py index d6e226ba968e..8fe4204a3746 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -809,7 +809,7 @@ def add_invertible_flag(flag: str, '--explicit-package-bases', action='store_true', help="Use current directory and MYPYPATH to determine module names of files passed") code_group.add_argument( - "--ignore-path", + "--exclude", metavar="PATH", action="append", default=[], diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index b2e6d1880289..9648890c20b4 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -451,7 +451,7 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: continue subpath = os.path.join(package_path, name) if self.options and any( - matches_ignore_pattern(subpath, pattern) for pattern in self.options.ignore_path + matches_exclude_pattern(subpath, pattern) for pattern in self.options.exclude ): continue @@ -475,7 +475,7 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: return sources -def matches_ignore_pattern(path: str, pattern: str) -> bool: +def matches_exclude_pattern(path: str, pattern: str) -> bool: path = os.path.splitdrive(path)[1] path_components = path.split(os.sep) pattern_components = pattern.replace(os.sep, "/").split("/") diff --git a/mypy/options.py b/mypy/options.py index 953335f8536e..a24cbd5d1c9e 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -98,7 +98,7 @@ def __init__(self) -> None: # top-level __init__.py to your packages. self.explicit_package_bases = False # File names, directory names or subpaths to avoid checking - self.ignore_path = [] # type: List[str] + self.exclude = [] # type: List[str] # disallow_any options self.disallow_any_generics = False diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index c8532af55738..e73c0f12985f 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -253,7 +253,7 @@ def test_find_sources_namespace_multi_dir(self) -> None: finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options) assert find_sources(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")] - def test_find_sources_ignore_path(self) -> None: + def test_find_sources_exclude(self) -> None: options = Options() options.namespace_packages = True @@ -270,7 +270,7 @@ def test_find_sources_ignore_path(self) -> None: } # file name - options.ignore_path = ["f.py"] + options.exclude = ["f.py"] finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), @@ -279,7 +279,7 @@ def test_find_sources_ignore_path(self) -> None: ] # directory name - options.ignore_path = ["a1"] + options.exclude = ["a1"] finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), @@ -288,7 +288,7 @@ def test_find_sources_ignore_path(self) -> None: ] # paths - options.ignore_path = ["/pkg/a1"] + options.exclude = ["/pkg/a1"] finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), @@ -296,7 +296,7 @@ def test_find_sources_ignore_path(self) -> None: ("a2.b.f", "/pkg"), ] - options.ignore_path = ["b/c"] + options.exclude = ["b/c"] finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), @@ -305,7 +305,7 @@ def test_find_sources_ignore_path(self) -> None: ] # nothing should be ignored as a result of this - options.ignore_path = [ + options.exclude = [ "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py" "xxx/pkg/a2/b/f.py" ] @@ -320,7 +320,7 @@ def test_find_sources_ignore_path(self) -> None: "pkg/a2/b/c/d/e.py", "pkg/a2/b/f.py", } - options.ignore_path = [ + options.exclude = [ "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py", "xxx/pkg/a2/b/f.py", "/pkg/a1", "/pkg/a2" ] diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 15fea22c7dc2..8902e77945fa 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -19,4 +19,4 @@ pretty = True always_false = MYPYC plugins = misc/proper_plugin.py python_version = 3.5 -ignore_path = mypy/typeshed +exclude = mypy/typeshed diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 87c3bd42b61f..55335a8d1dbe 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -59,7 +59,7 @@ undef undef [out] dir/a.py: error: Duplicate module named 'a' (also at 'dir/subdir/a.py') -dir/a.py: note: Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid checking one of them. +dir/a.py: note: Are you missing an __init__.py? Alternatively, consider using --exclude to avoid checking one of them. == Return code: 2 [case testCmdlineNonPackageSlash] @@ -125,7 +125,7 @@ mypy: can't decode file 'a.py': unknown encoding: uft-8 # type: ignore [out] two/mod/__init__.py: error: Duplicate module named 'mod' (also at 'one/mod/__init__.py') -two/mod/__init__.py: note: Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid checking one of them. +two/mod/__init__.py: note: Are you missing an __init__.py? Alternatively, consider using --exclude to avoid checking one of them. == Return code: 2 [case testFlagsFile] From 7e92e9bdba8cfe9a9ce7e037b2effd6e70577b33 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Mon, 1 Feb 2021 22:17:48 -0800 Subject: [PATCH 10/22] use regexes for exclude --- docs/source/command_line.rst | 6 +++--- docs/source/config_file.rst | 7 ++++--- mypy/build.py | 1 + mypy/config_parser.py | 1 - mypy/find_sources.py | 19 +++++++++---------- mypy/main.py | 11 ++++++----- mypy/modulefinder.py | 24 ++++++++---------------- mypy/options.py | 2 +- mypy/test/test_find_sources.py | 23 +++++++++-------------- mypy_self_check.ini | 2 +- 10 files changed, 42 insertions(+), 54 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 6747ff46269e..8151e4e7d82e 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -51,9 +51,9 @@ for full details, see :ref:`running-mypy`. .. option:: --exclude - Asks mypy to exclude a given file name, directory name or subpath while - recursively discovering files to check. This flag may be repeated multiple - times. + A regular expression that matches file names, directory names and paths + which mypy should ignore while recursively discovering files to check. + Use forward slashes on all platforms. Optional arguments diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index d7562531e74c..8ec826f709e6 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -199,10 +199,11 @@ section of the command line docs. .. confval:: exclude - :type: comma-separated list of strings + :type: regular expression - A comma-separated list of file names, directory names or subpaths which mypy - should ignore while recursively discovering files to check. + A regular expression that matches file names, directory names and paths + which mypy should ignore while recursively discovering files to check. + Use forward slashes on all platforms. This option may only be set in the global section (``[mypy]``). diff --git a/mypy/build.py b/mypy/build.py index 1aad548a6e8c..cff1c9cff949 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2569,6 +2569,7 @@ def log_configuration(manager: BuildManager, sources: List[BuildSource]) -> None ("Current Executable", sys.executable), ("Cache Dir", manager.options.cache_dir), ("Compiled", str(not __file__.endswith(".py"))), + ("Exclude", manager.options.exclude), ] for conf_name, conf_value in configuration_vars: diff --git a/mypy/config_parser.py b/mypy/config_parser.py index 5d116ace41d1..dd79869030e5 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -83,7 +83,6 @@ def check_follow_imports(choice: str) -> str: 'custom_typing_module': str, 'custom_typeshed_dir': expand_path, 'mypy_path': lambda s: [expand_path(p.strip()) for p in re.split('[,:]', s)], - 'exclude': lambda s: [expand_path(p.strip()) for p in s.split(",")], 'files': split_and_match_files, 'quickstart_file': expand_path, 'junit_xml': expand_path, diff --git a/mypy/find_sources.py b/mypy/find_sources.py index 4adaf31e8915..af9c5ba3b876 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -2,11 +2,12 @@ import functools import os +import re from typing import List, Sequence, Set, Tuple, Optional from typing_extensions import Final -from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path, matches_exclude_pattern +from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path from mypy.fscache import FileSystemCache from mypy.options import Options @@ -103,16 +104,14 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]: seen = set() # type: Set[str] names = sorted(self.fscache.listdir(path), key=keyfunc) for name in names: - # Skip certain names altogether - if ( - name in ("__pycache__", "site-packages", "node_modules") - or name.startswith(".") - or name.endswith("~") - ): - continue subpath = os.path.join(path, name) - if any(matches_exclude_pattern(subpath, pattern) for pattern in self.exclude): - continue + + if self.exclude: + subpath_str = "/" + os.path.abspath(subpath).replace(os.sep, "/") + if self.fscache.isdir(subpath): + subpath_str += "/" + if re.search(self.exclude, subpath_str): + continue if self.fscache.isdir(subpath): sub_sources = self.find_sources_in_dir(subpath) diff --git a/mypy/main.py b/mypy/main.py index 8fe4204a3746..c5b0da9c25bd 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -424,6 +424,7 @@ def add_invertible_flag(flag: str, # Options object. Options that require further processing should have # their `dest` prefixed with `special-opts:`, which will cause them to be # parsed into the separate special_opts namespace object. + options = Options() # Note: we have a style guide for formatting the mypy --help text. See # https://github.com/python/mypy/wiki/Documentation-Conventions @@ -811,9 +812,11 @@ def add_invertible_flag(flag: str, code_group.add_argument( "--exclude", metavar="PATH", - action="append", - default=[], - help="File names, directory names or subpaths to avoid checking", + default=options.exclude, + help=( + "Regex to match file names, directory names or paths to avoid checking. " + "Defaults to '%(default)s'." + ) ) code_group.add_argument( '-m', '--module', action='append', metavar='MODULE', @@ -843,8 +846,6 @@ def add_invertible_flag(flag: str, if config_file and not os.path.exists(config_file): parser.error("Cannot find config file '%s'" % config_file) - options = Options() - def set_strict_flags() -> None: for dest, value in strict_flag_assignments: setattr(options, dest, value) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 9648890c20b4..9de86d50e1cb 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -7,6 +7,7 @@ import collections import functools import os +import re import subprocess import sys from enum import Enum @@ -450,10 +451,13 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: ): continue subpath = os.path.join(package_path, name) - if self.options and any( - matches_exclude_pattern(subpath, pattern) for pattern in self.options.exclude - ): - continue + + if self.options and self.options.exclude: + subpath_str = "/" + os.path.abspath(subpath).replace(os.sep, "/") + if self.fscache.isdir(subpath): + subpath_str += "/" + if re.search(self.options.exclude, subpath_str): + continue if self.fscache.isdir(subpath): # Only recurse into packages @@ -475,18 +479,6 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: return sources -def matches_exclude_pattern(path: str, pattern: str) -> bool: - path = os.path.splitdrive(path)[1] - path_components = path.split(os.sep) - pattern_components = pattern.replace(os.sep, "/").split("/") - if len(path_components) < len(pattern_components): - return False - return all( - path == pattern - for path, pattern in zip(reversed(path_components), reversed(pattern_components)) - ) - - def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> bool: """Check that all packages containing id have a __init__ file.""" if path.endswith(('__init__.py', '__init__.pyi')): diff --git a/mypy/options.py b/mypy/options.py index a24cbd5d1c9e..b9441e2353d3 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -98,7 +98,7 @@ def __init__(self) -> None: # top-level __init__.py to your packages. self.explicit_package_bases = False # File names, directory names or subpaths to avoid checking - self.exclude = [] # type: List[str] + self.exclude = "/(__pycache__|site-packages|node_modules)/|/(\\.[^/]+|[^/]+~)/?$" # type: str # disallow_any options self.disallow_any_generics = False diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index e73c0f12985f..2fc002b1ee9d 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -257,7 +257,7 @@ def test_find_sources_exclude(self) -> None: options = Options() options.namespace_packages = True - # special cased name + # default finder = SourceFinder(FakeFSCache({"/dir/a.py", "/dir/venv/site-packages/b.py"}), options) assert find_sources(finder, "/") == [("a", "/dir")] @@ -270,7 +270,7 @@ def test_find_sources_exclude(self) -> None: } # file name - options.exclude = ["f.py"] + options.exclude = "/f.py" finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), @@ -279,7 +279,7 @@ def test_find_sources_exclude(self) -> None: ] # directory name - options.exclude = ["a1"] + options.exclude = "/a1/" finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), @@ -288,7 +288,7 @@ def test_find_sources_exclude(self) -> None: ] # paths - options.exclude = ["/pkg/a1"] + options.exclude = "/pkg/a1/" finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), @@ -296,7 +296,7 @@ def test_find_sources_exclude(self) -> None: ("a2.b.f", "/pkg"), ] - options.exclude = ["b/c"] + options.exclude = "b/c/" finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), @@ -305,14 +305,13 @@ def test_find_sources_exclude(self) -> None: ] # nothing should be ignored as a result of this - options.exclude = [ - "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py" - "xxx/pkg/a2/b/f.py" - ] + options.exclude = "|".join(( + "/pkg/a/", "/2", "/1", "/pk/", "/kg", "/g.py", "/bc", "/xxx/pkg/a2/b/f.py" + "xxx/pkg/a2/b/f.py", + )) finder = SourceFinder(FakeFSCache(files), options) assert len(find_sources(finder, "/")) == len(files) - # nothing should be ignored as a result of this files = { "pkg/a1/b/c/d/e.py", "pkg/a1/b/f.py", @@ -320,9 +319,5 @@ def test_find_sources_exclude(self) -> None: "pkg/a2/b/c/d/e.py", "pkg/a2/b/f.py", } - options.exclude = [ - "/pkg/a", "2", "1", "pk", "kg", "g.py", "bc", "/b", "/xxx/pkg/a2/b/f.py", - "xxx/pkg/a2/b/f.py", "/pkg/a1", "/pkg/a2" - ] finder = SourceFinder(FakeFSCache(files), options) assert len(find_sources(finder, "/")) == len(files) diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 8902e77945fa..ad041dc59f83 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -19,4 +19,4 @@ pretty = True always_false = MYPYC plugins = misc/proper_plugin.py python_version = 3.5 -exclude = mypy/typeshed +exclude = /(mypy/typeshed|__pycache__|site-packages|node_modules)/|/(\.[^/]+|[^/]+~)/?$ From da26d923110ce867dc8fbd9684b6b0992c83f9ca Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Mon, 1 Feb 2021 22:47:49 -0800 Subject: [PATCH 11/22] add trace logs for excluded dirs --- mypy/find_sources.py | 6 +++++- mypy/modulefinder.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mypy/find_sources.py b/mypy/find_sources.py index af9c5ba3b876..df70411dd7c8 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -3,6 +3,7 @@ import functools import os import re +import sys from typing import List, Sequence, Set, Tuple, Optional from typing_extensions import Final @@ -93,6 +94,7 @@ def __init__(self, fscache: FileSystemCache, options: Options) -> None: self.explicit_package_bases = get_explicit_package_bases(options) self.namespace_packages = options.namespace_packages self.exclude = options.exclude + self.verbose = options.verbosity >= 2 def is_explicit_package_base(self, path: str) -> bool: assert self.explicit_package_bases @@ -107,10 +109,12 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]: subpath = os.path.join(path, name) if self.exclude: - subpath_str = "/" + os.path.abspath(subpath).replace(os.sep, "/") + subpath_str = os.path.abspath(subpath).replace(os.sep, "/") if self.fscache.isdir(subpath): subpath_str += "/" if re.search(self.exclude, subpath_str): + if self.verbose: + print("TRACE: Excluding {}".format(subpath_str), file=sys.stderr) continue if self.fscache.isdir(subpath): diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 9de86d50e1cb..379ec730a303 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -453,10 +453,12 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: subpath = os.path.join(package_path, name) if self.options and self.options.exclude: - subpath_str = "/" + os.path.abspath(subpath).replace(os.sep, "/") + subpath_str = os.path.abspath(subpath).replace(os.sep, "/") if self.fscache.isdir(subpath): subpath_str += "/" if re.search(self.options.exclude, subpath_str): + if self.options.verbosity >= 2: + print("TRACE: Excluding {}".format(subpath_str), file=sys.stderr) continue if self.fscache.isdir(subpath): From d470d5c7054ea7d77ffb1aba4702358b9bcd8fbc Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Mon, 1 Feb 2021 22:48:13 -0800 Subject: [PATCH 12/22] simplify exclude rule --- mypy_self_check.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy_self_check.ini b/mypy_self_check.ini index ad041dc59f83..5c373725abfa 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -19,4 +19,4 @@ pretty = True always_false = MYPYC plugins = misc/proper_plugin.py python_version = 3.5 -exclude = /(mypy/typeshed|__pycache__|site-packages|node_modules)/|/(\.[^/]+|[^/]+~)/?$ +exclude = /(mypy/typeshed|__pycache__|site-packages)/|/(\.[^/]+)/?$ From 1c0b5c33729eac3bb3d8d051f45433e3ed2a7798 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Mon, 1 Feb 2021 22:50:30 -0800 Subject: [PATCH 13/22] flake8 --- mypy/options.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/options.py b/mypy/options.py index b9441e2353d3..6eda4cfe92ce 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -98,7 +98,9 @@ def __init__(self) -> None: # top-level __init__.py to your packages. self.explicit_package_bases = False # File names, directory names or subpaths to avoid checking - self.exclude = "/(__pycache__|site-packages|node_modules)/|/(\\.[^/]+|[^/]+~)/?$" # type: str + self.exclude = ( + "/(__pycache__|site-packages|node_modules)/|/(\\.[^/]+|[^/]+~)/?$" + ) # type: str # disallow_any options self.disallow_any_generics = False From 6254323191dffdb548e4b2ab9659f6b4549f20ea Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 2 Feb 2021 18:35:27 -0800 Subject: [PATCH 14/22] don't override defaults --- mypy/find_sources.py | 21 +++++++++------------ mypy/main.py | 9 +++++---- mypy/modulefinder.py | 35 ++++++++++++++++++++--------------- mypy/options.py | 4 +--- mypy_self_check.ini | 2 +- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/mypy/find_sources.py b/mypy/find_sources.py index df70411dd7c8..7f021f3ab611 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -2,13 +2,11 @@ import functools import os -import re -import sys from typing import List, Sequence, Set, Tuple, Optional from typing_extensions import Final -from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path +from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path, matches_exclude from mypy.fscache import FileSystemCache from mypy.options import Options @@ -94,7 +92,7 @@ def __init__(self, fscache: FileSystemCache, options: Options) -> None: self.explicit_package_bases = get_explicit_package_bases(options) self.namespace_packages = options.namespace_packages self.exclude = options.exclude - self.verbose = options.verbosity >= 2 + self.verbosity = options.verbosity def is_explicit_package_base(self, path: str) -> bool: assert self.explicit_package_bases @@ -106,16 +104,15 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]: seen = set() # type: Set[str] names = sorted(self.fscache.listdir(path), key=keyfunc) for name in names: + # Skip certain names altogether + if name in ("__pycache__", "site-packages") or name.startswith("."): + continue subpath = os.path.join(path, name) - if self.exclude: - subpath_str = os.path.abspath(subpath).replace(os.sep, "/") - if self.fscache.isdir(subpath): - subpath_str += "/" - if re.search(self.exclude, subpath_str): - if self.verbose: - print("TRACE: Excluding {}".format(subpath_str), file=sys.stderr) - continue + if matches_exclude( + subpath, self.exclude, self.fscache, self.verbosity >= 2 + ): + continue if self.fscache.isdir(subpath): sub_sources = self.find_sources_in_dir(subpath) diff --git a/mypy/main.py b/mypy/main.py index c5b0da9c25bd..eec259254107 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -424,7 +424,6 @@ def add_invertible_flag(flag: str, # Options object. Options that require further processing should have # their `dest` prefixed with `special-opts:`, which will cause them to be # parsed into the separate special_opts namespace object. - options = Options() # Note: we have a style guide for formatting the mypy --help text. See # https://github.com/python/mypy/wiki/Documentation-Conventions @@ -812,10 +811,10 @@ def add_invertible_flag(flag: str, code_group.add_argument( "--exclude", metavar="PATH", - default=options.exclude, + default="", help=( - "Regex to match file names, directory names or paths to avoid checking. " - "Defaults to '%(default)s'." + "Regular expression to match file names, directory names or paths which mypy should " + "ignore while recursively discovering files to check." ) ) code_group.add_argument( @@ -846,6 +845,8 @@ def add_invertible_flag(flag: str, if config_file and not os.path.exists(config_file): parser.error("Cannot find config file '%s'" % config_file) + options = Options() + def set_strict_flags() -> None: for dest, value in strict_flag_assignments: setattr(options, dest, value) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 379ec730a303..2b2c5e21cd35 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -444,22 +444,14 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: names = sorted(self.fscache.listdir(package_path)) for name in names: # Skip certain names altogether - if ( - name in ("__pycache__", "site-packages", "node_modules") - or name.startswith(".") - or name.endswith("~") - ): + if name in ("__pycache__", "site-packages") or name.startswith("."): continue subpath = os.path.join(package_path, name) - if self.options and self.options.exclude: - subpath_str = os.path.abspath(subpath).replace(os.sep, "/") - if self.fscache.isdir(subpath): - subpath_str += "/" - if re.search(self.options.exclude, subpath_str): - if self.options.verbosity >= 2: - print("TRACE: Excluding {}".format(subpath_str), file=sys.stderr) - continue + if self.options and matches_exclude( + subpath, self.options.exclude, self.fscache, self.options.verbosity >= 2 + ): + continue if self.fscache.isdir(subpath): # Only recurse into packages @@ -474,13 +466,26 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: if stem == '__init__': continue if stem not in seen and '.' not in stem and suffix in PYTHON_EXTENSIONS: - # (If we sorted names) we could probably just make the BuildSource ourselves, - # but this ensures compatibility with find_module / the cache + # (If we sorted names by keyfunc) we could probably just make the BuildSource + # ourselves, but this ensures compatibility with find_module / the cache seen.add(stem) sources.extend(self.find_modules_recursive(module + '.' + stem)) return sources +def matches_exclude(subpath: str, exclude: str, fscache: FileSystemCache, verbose: bool) -> bool: + if not exclude: + return False + subpath_str = os.path.abspath(subpath).replace(os.sep, "/") + if fscache.isdir(subpath): + subpath_str += "/" + if re.search(exclude, subpath_str): + if verbose >= 2: + print("TRACE: Excluding {}".format(subpath_str), file=sys.stderr) + return True + return False + + def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> bool: """Check that all packages containing id have a __init__ file.""" if path.endswith(('__init__.py', '__init__.pyi')): diff --git a/mypy/options.py b/mypy/options.py index 6eda4cfe92ce..0d19a4046245 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -98,9 +98,7 @@ def __init__(self) -> None: # top-level __init__.py to your packages. self.explicit_package_bases = False # File names, directory names or subpaths to avoid checking - self.exclude = ( - "/(__pycache__|site-packages|node_modules)/|/(\\.[^/]+|[^/]+~)/?$" - ) # type: str + self.exclude = "" # type: str # disallow_any options self.disallow_any_generics = False diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 5c373725abfa..c974a0248afc 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -19,4 +19,4 @@ pretty = True always_false = MYPYC plugins = misc/proper_plugin.py python_version = 3.5 -exclude = /(mypy/typeshed|__pycache__|site-packages)/|/(\.[^/]+)/?$ +exclude = /mypy/typeshed/ From a4ae3aeda524c93ae7da7b3db19d5ea3ebffa3a3 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 2 Feb 2021 19:01:26 -0800 Subject: [PATCH 15/22] docs and tests --- docs/source/command_line.rst | 16 ++++++++++++++++ docs/source/config_file.rst | 2 ++ mypy/test/test_find_sources.py | 12 +++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 8151e4e7d82e..268dcfea7fdf 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -55,6 +55,22 @@ for full details, see :ref:`running-mypy`. which mypy should ignore while recursively discovering files to check. Use forward slashes on all platforms. + For instance, to avoid discovering any files named `setup.py` you could + pass ``--exclude '/setup.py$'``. Similarly, you can ignore discovering + directories with a given name by e.g. ``--exclude /node_modules/`` or + those matching a subpath with ``--exclude /project/vendor/``. + + Note that this flag only affects recursive discovery, that is, when mypy is + discovering files within a directory tree or submodules of a package to + check. If you pass a file or module explicitly it will still be checked. For + instance, ``mypy --exclude '/setup.py$' but_still_check/setup.py``. + + Note that mypy will never recursively discover files and directories named + "site-packages" or "__pycache__" or those whose name starts with a period, + exactly as ``--exclude '/(site-packages|__pycache__|\..*)$'`` would. + Mypy will also never recursively discover files with extensions other than + ``.py`` or ``.pyi``. + Optional arguments ****************** diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index 8ec826f709e6..5f7097aebe2a 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -205,6 +205,8 @@ section of the command line docs. which mypy should ignore while recursively discovering files to check. Use forward slashes on all platforms. + For more details, see :option:`--exclude `. + This option may only be set in the global section (``[mypy]``). .. confval:: namespace_packages diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 2fc002b1ee9d..422b8076463d 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -260,6 +260,8 @@ def test_find_sources_exclude(self) -> None: # default finder = SourceFinder(FakeFSCache({"/dir/a.py", "/dir/venv/site-packages/b.py"}), options) assert find_sources(finder, "/") == [("a", "/dir")] + assert find_sources(finder, "/dir/venv/") == [] + assert find_sources(finder, "/dir/venv/site-packages") == [('b', '/dir/venv/site-packages')] files = { "/pkg/a1/b/c/d/e.py", @@ -270,7 +272,7 @@ def test_find_sources_exclude(self) -> None: } # file name - options.exclude = "/f.py" + options.exclude = "/f.py$" finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ ("a2", "/pkg"), @@ -296,6 +298,14 @@ def test_find_sources_exclude(self) -> None: ("a2.b.f", "/pkg"), ] + options.exclude = "/(a1|a3)/" + finder = SourceFinder(FakeFSCache(files), options) + assert find_sources(finder, "/") == [ + ("a2", "/pkg"), + ("a2.b.c.d.e", "/pkg"), + ("a2.b.f", "/pkg"), + ] + options.exclude = "b/c/" finder = SourceFinder(FakeFSCache(files), options) assert find_sources(finder, "/") == [ From 59af7098c7f1f66ba3bdfe1cd3b639dd271141a1 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 2 Feb 2021 19:36:10 -0800 Subject: [PATCH 16/22] refactor tests, add some cases --- mypy/test/test_find_sources.py | 86 ++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 29 deletions(-) diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 422b8076463d..3e512f2d5e53 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -1,8 +1,9 @@ from mypy.modulefinder import BuildSource import os +import pytest import unittest from typing import List, Optional, Set, Tuple -from mypy.find_sources import SourceFinder +from mypy.find_sources import InvalidSourceList, SourceFinder, create_source_list from mypy.fscache import FileSystemCache from mypy.modulefinder import BuildSource from mypy.options import Options @@ -47,10 +48,16 @@ def crawl(finder: SourceFinder, f: str) -> Tuple[str, str]: return module, normalise_path(base_dir) -def find_sources(finder: SourceFinder, f: str) -> List[Tuple[str, Optional[str]]]: +def find_sources_in_dir(finder: SourceFinder, f: str) -> List[Tuple[str, Optional[str]]]: return normalise_build_source_list(finder.find_sources_in_dir(os.path.abspath(f))) +def find_sources( + paths: List[str], options: Options, fscache: FileSystemCache +) -> List[Tuple[str, Optional[str]]]: + return normalise_build_source_list(create_source_list(paths, options, fscache)) + + class SourceFinderSuite(unittest.TestCase): def test_crawl_no_namespace(self) -> None: options = Options() @@ -172,7 +179,7 @@ def test_crawl_namespace_multi_dir(self) -> None: assert crawl(finder, "/a/pkg/a.py") == ("pkg.a", "/a") assert crawl(finder, "/b/pkg/b.py") == ("pkg.b", "/b") - def test_find_sources_no_namespace(self) -> None: + def test_find_sources_in_dir_no_namespace(self) -> None: options = Options() options.namespace_packages = False @@ -184,7 +191,7 @@ def test_find_sources_no_namespace(self) -> None: "/pkg/a2/b/f.py", } finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + assert find_sources_in_dir(finder, "/") == [ ("a2", "/pkg"), ("e", "/pkg/a1/b/c/d"), ("e", "/pkg/a2/b/c/d"), @@ -192,7 +199,7 @@ def test_find_sources_no_namespace(self) -> None: ("f", "/pkg/a2/b"), ] - def test_find_sources_namespace(self) -> None: + def test_find_sources_in_dir_namespace(self) -> None: options = Options() options.namespace_packages = True @@ -204,7 +211,7 @@ def test_find_sources_namespace(self) -> None: "/pkg/a2/b/f.py", } finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + assert find_sources_in_dir(finder, "/") == [ ("a2", "/pkg"), ("a2.b.c.d.e", "/pkg"), ("a2.b.f", "/pkg"), @@ -212,7 +219,7 @@ def test_find_sources_namespace(self) -> None: ("f", "/pkg/a1/b"), ] - def test_find_sources_namespace_explicit_base(self) -> None: + def test_find_sources_in_dir_namespace_explicit_base(self) -> None: options = Options() options.namespace_packages = True options.explicit_package_bases = True @@ -226,7 +233,7 @@ def test_find_sources_namespace_explicit_base(self) -> None: "/pkg/a2/b/f.py", } finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + assert find_sources_in_dir(finder, "/") == [ ("pkg.a1.b.c.d.e", "/"), ("pkg.a1.b.f", "/"), ("pkg.a2", "/"), @@ -236,7 +243,7 @@ def test_find_sources_namespace_explicit_base(self) -> None: options.mypy_path = ["/pkg"] finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + assert find_sources_in_dir(finder, "/") == [ ("a1.b.c.d.e", "/pkg"), ("a1.b.f", "/pkg"), ("a2", "/pkg"), @@ -244,24 +251,30 @@ def test_find_sources_namespace_explicit_base(self) -> None: ("a2.b.f", "/pkg"), ] - def test_find_sources_namespace_multi_dir(self) -> None: + def test_find_sources_in_dir_namespace_multi_dir(self) -> None: options = Options() options.namespace_packages = True options.explicit_package_bases = True options.mypy_path = ["/a", "/b"] finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options) - assert find_sources(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")] + assert find_sources_in_dir(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")] def test_find_sources_exclude(self) -> None: options = Options() options.namespace_packages = True # default - finder = SourceFinder(FakeFSCache({"/dir/a.py", "/dir/venv/site-packages/b.py"}), options) - assert find_sources(finder, "/") == [("a", "/dir")] - assert find_sources(finder, "/dir/venv/") == [] - assert find_sources(finder, "/dir/venv/site-packages") == [('b', '/dir/venv/site-packages')] + fscache = FakeFSCache({"/dir/a.py", "/dir/venv/site-packages/b.py"}) + assert find_sources(["/"], options, fscache) == [("a", "/dir")] + with pytest.raises(InvalidSourceList): + find_sources(["/dir/venv/"], options, fscache) + assert find_sources(["/dir/venv/site-packages"], options, fscache) == [ + ("b", "/dir/venv/site-packages") + ] + assert find_sources(["/dir/venv/site-packages/b.py"], options, fscache) == [ + ("b", "/dir/venv/site-packages") + ] files = { "/pkg/a1/b/c/d/e.py", @@ -273,42 +286,57 @@ def test_find_sources_exclude(self) -> None: # file name options.exclude = "/f.py$" - finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ ("a2", "/pkg"), ("a2.b.c.d.e", "/pkg"), ("e", "/pkg/a1/b/c/d"), ] + assert find_sources(["/pkg/a1/b/f.py"], options, fscache) == [('f', '/pkg/a1/b')] + assert find_sources(["/pkg/a2/b/f.py"], options, fscache) == [('a2.b.f', '/pkg')] # directory name options.exclude = "/a1/" - finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ ("a2", "/pkg"), ("a2.b.c.d.e", "/pkg"), ("a2.b.f", "/pkg"), ] + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1"], options, fscache) + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1/"], options, fscache) + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1/b"], options, fscache) + + options.exclude = "/a1/$" + assert find_sources(["/pkg/a1"], options, fscache) == [ + ('e', '/pkg/a1/b/c/d'), ('f', '/pkg/a1/b') + ] # paths options.exclude = "/pkg/a1/" - finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ ("a2", "/pkg"), ("a2.b.c.d.e", "/pkg"), ("a2.b.f", "/pkg"), ] + with pytest.raises(InvalidSourceList): + find_sources(["/pkg/a1"], options, fscache) options.exclude = "/(a1|a3)/" - finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ ("a2", "/pkg"), ("a2.b.c.d.e", "/pkg"), ("a2.b.f", "/pkg"), ] options.exclude = "b/c/" - finder = SourceFinder(FakeFSCache(files), options) - assert find_sources(finder, "/") == [ + fscache = FakeFSCache(files) + assert find_sources(["/"], options, fscache) == [ ("a2", "/pkg"), ("a2.b.f", "/pkg"), ("f", "/pkg/a1/b"), @@ -319,8 +347,8 @@ def test_find_sources_exclude(self) -> None: "/pkg/a/", "/2", "/1", "/pk/", "/kg", "/g.py", "/bc", "/xxx/pkg/a2/b/f.py" "xxx/pkg/a2/b/f.py", )) - finder = SourceFinder(FakeFSCache(files), options) - assert len(find_sources(finder, "/")) == len(files) + fscache = FakeFSCache(files) + assert len(find_sources(["/"], options, fscache)) == len(files) files = { "pkg/a1/b/c/d/e.py", @@ -329,5 +357,5 @@ def test_find_sources_exclude(self) -> None: "pkg/a2/b/c/d/e.py", "pkg/a2/b/f.py", } - finder = SourceFinder(FakeFSCache(files), options) - assert len(find_sources(finder, "/")) == len(files) + fscache = FakeFSCache(files) + assert len(find_sources(["/"], options, fscache)) == len(files) From f5be793e4888178940fb8e18135869ee33364a63 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 2 Feb 2021 19:42:53 -0800 Subject: [PATCH 17/22] fix verbose --- mypy/modulefinder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 2b2c5e21cd35..b2effc461a22 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -480,7 +480,7 @@ def matches_exclude(subpath: str, exclude: str, fscache: FileSystemCache, verbos if fscache.isdir(subpath): subpath_str += "/" if re.search(exclude, subpath_str): - if verbose >= 2: + if verbose: print("TRACE: Excluding {}".format(subpath_str), file=sys.stderr) return True return False From cb0917d28042891f72c80b987425b8f88d51573e Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 2 Feb 2021 19:49:56 -0800 Subject: [PATCH 18/22] add node_modules back to default exclusion --- docs/source/command_line.rst | 11 ++++++----- mypy/find_sources.py | 2 +- mypy/modulefinder.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 268dcfea7fdf..8729968fd857 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -57,7 +57,7 @@ for full details, see :ref:`running-mypy`. For instance, to avoid discovering any files named `setup.py` you could pass ``--exclude '/setup.py$'``. Similarly, you can ignore discovering - directories with a given name by e.g. ``--exclude /node_modules/`` or + directories with a given name by e.g. ``--exclude /build/`` or those matching a subpath with ``--exclude /project/vendor/``. Note that this flag only affects recursive discovery, that is, when mypy is @@ -66,10 +66,11 @@ for full details, see :ref:`running-mypy`. instance, ``mypy --exclude '/setup.py$' but_still_check/setup.py``. Note that mypy will never recursively discover files and directories named - "site-packages" or "__pycache__" or those whose name starts with a period, - exactly as ``--exclude '/(site-packages|__pycache__|\..*)$'`` would. - Mypy will also never recursively discover files with extensions other than - ``.py`` or ``.pyi``. + "site-packages", "node_modules" or "__pycache__", or those whose name starts + with a period, exactly as ``--exclude + '/(site-packages|node_modules|__pycache__|\..*)$'`` would. Mypy will also + never recursively discover files with extensions other than ``.py`` or + ``.pyi``. Optional arguments diff --git a/mypy/find_sources.py b/mypy/find_sources.py index 7f021f3ab611..4f50d8ff52b2 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -105,7 +105,7 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]: names = sorted(self.fscache.listdir(path), key=keyfunc) for name in names: # Skip certain names altogether - if name in ("__pycache__", "site-packages") or name.startswith("."): + if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."): continue subpath = os.path.join(path, name) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index b2effc461a22..00a0530e29d5 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -444,7 +444,7 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: names = sorted(self.fscache.listdir(package_path)) for name in names: # Skip certain names altogether - if name in ("__pycache__", "site-packages") or name.startswith("."): + if name in ("__pycache__", "site-packages", "node_modules") or name.startswith("."): continue subpath = os.path.join(package_path, name) From 43d8827a35ff67bc65c68ab64ec374cd669aa20a Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 2 Feb 2021 19:54:14 -0800 Subject: [PATCH 19/22] fix a small lie --- docs/source/command_line.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 8729968fd857..7378e4decd4a 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -68,7 +68,7 @@ for full details, see :ref:`running-mypy`. Note that mypy will never recursively discover files and directories named "site-packages", "node_modules" or "__pycache__", or those whose name starts with a period, exactly as ``--exclude - '/(site-packages|node_modules|__pycache__|\..*)$'`` would. Mypy will also + '/(site-packages|node_modules|__pycache__|\..*)/$'`` would. Mypy will also never recursively discover files with extensions other than ``.py`` or ``.pyi``. From bdf3f9afc5bb0f2da753438a945b5c7d1f3d3fb0 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 2 Feb 2021 21:51:38 -0800 Subject: [PATCH 20/22] fix flake8, try and fix windows --- mypy/options.py | 2 +- mypy/test/test_find_sources.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/options.py b/mypy/options.py index 0d19a4046245..026046af5da4 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -98,7 +98,7 @@ def __init__(self) -> None: # top-level __init__.py to your packages. self.explicit_package_bases = False # File names, directory names or subpaths to avoid checking - self.exclude = "" # type: str + self.exclude = "" # type: str # disallow_any options self.disallow_any_generics = False diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 3e512f2d5e53..f8aa9f6f4af7 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -55,6 +55,7 @@ def find_sources_in_dir(finder: SourceFinder, f: str) -> List[Tuple[str, Optiona def find_sources( paths: List[str], options: Options, fscache: FileSystemCache ) -> List[Tuple[str, Optional[str]]]: + paths = [os.path.abspath(p) for p in paths] return normalise_build_source_list(create_source_list(paths, options, fscache)) From 962153d700e96c2dfa851ce7c6bea235fc18ab22 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Wed, 3 Feb 2021 11:55:02 -0800 Subject: [PATCH 21/22] improve help text --- mypy/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index eec259254107..00546c83833c 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -810,11 +810,11 @@ def add_invertible_flag(flag: str, help="Use current directory and MYPYPATH to determine module names of files passed") code_group.add_argument( "--exclude", - metavar="PATH", + metavar="PATTERN", default="", help=( "Regular expression to match file names, directory names or paths which mypy should " - "ignore while recursively discovering files to check." + "ignore while recursively discovering files to check, e.g. --exclude '/setup.py$'" ) ) code_group.add_argument( From 2abe732a3505551092ef251ab59cdc3551027b84 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 5 Feb 2021 22:13:40 -0800 Subject: [PATCH 22/22] nits --- docs/source/command_line.rst | 2 +- mypy/main.py | 2 +- mypy/test/test_find_sources.py | 23 ++++++++++++----------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 7378e4decd4a..0161dd5eaada 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -56,7 +56,7 @@ for full details, see :ref:`running-mypy`. Use forward slashes on all platforms. For instance, to avoid discovering any files named `setup.py` you could - pass ``--exclude '/setup.py$'``. Similarly, you can ignore discovering + pass ``--exclude '/setup\.py$'``. Similarly, you can ignore discovering directories with a given name by e.g. ``--exclude /build/`` or those matching a subpath with ``--exclude /project/vendor/``. diff --git a/mypy/main.py b/mypy/main.py index 00546c83833c..d4ee3c190b49 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -814,7 +814,7 @@ def add_invertible_flag(flag: str, default="", help=( "Regular expression to match file names, directory names or paths which mypy should " - "ignore while recursively discovering files to check, e.g. --exclude '/setup.py$'" + "ignore while recursively discovering files to check, e.g. --exclude '/setup\\.py$'" ) ) code_group.add_argument( diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index f8aa9f6f4af7..056ddf13b108 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -266,16 +266,17 @@ def test_find_sources_exclude(self) -> None: options.namespace_packages = True # default - fscache = FakeFSCache({"/dir/a.py", "/dir/venv/site-packages/b.py"}) - assert find_sources(["/"], options, fscache) == [("a", "/dir")] - with pytest.raises(InvalidSourceList): - find_sources(["/dir/venv/"], options, fscache) - assert find_sources(["/dir/venv/site-packages"], options, fscache) == [ - ("b", "/dir/venv/site-packages") - ] - assert find_sources(["/dir/venv/site-packages/b.py"], options, fscache) == [ - ("b", "/dir/venv/site-packages") - ] + for excluded_dir in ["site-packages", ".whatever", "node_modules", ".x/.z"]: + fscache = FakeFSCache({"/dir/a.py", "/dir/venv/{}/b.py".format(excluded_dir)}) + assert find_sources(["/"], options, fscache) == [("a", "/dir")] + with pytest.raises(InvalidSourceList): + find_sources(["/dir/venv/"], options, fscache) + assert find_sources(["/dir/venv/{}".format(excluded_dir)], options, fscache) == [ + ("b", "/dir/venv/{}".format(excluded_dir)) + ] + assert find_sources(["/dir/venv/{}/b.py".format(excluded_dir)], options, fscache) == [ + ("b", "/dir/venv/{}".format(excluded_dir)) + ] files = { "/pkg/a1/b/c/d/e.py", @@ -286,7 +287,7 @@ def test_find_sources_exclude(self) -> None: } # file name - options.exclude = "/f.py$" + options.exclude = r"/f\.py$" fscache = FakeFSCache(files) assert find_sources(["/"], options, fscache) == [ ("a2", "/pkg"),