From 513b7e18109de24cf21157dd93b7969a9b5b0cd9 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Thu, 28 Jan 2021 23:29:13 -0800 Subject: [PATCH] Add --ignore-path Resolves #4675, resolves #9981 Additionally, we always ignore site-packages and node_modules. Also note that this doesn't really affect import discovery; it only directly affects passing files or packages to mypy. The additional check before suggesting "are you missing an __init__.py" didn't make any sense to me, so I removed it, appended to the message and downgraded the severity to note. --- docs/source/command_line.rst | 7 +++++++ docs/source/config_file.rst | 9 +++++++++ docs/source/running_mypy.rst | 3 ++- mypy/build.py | 14 ++++++-------- mypy/config_parser.py | 1 + mypy/find_sources.py | 9 ++++++++- mypy/main.py | 7 +++++++ mypy/modulefinder.py | 10 +++++++++- mypy/options.py | 2 ++ mypy/test/test_find_sources.py | 21 +++++++++++++++++++++ mypy_self_check.ini | 1 + test-data/unit/cmdline.test | 16 ++-------------- 12 files changed, 75 insertions(+), 25 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 8b8e7e6ee928b..e59027ac41927 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -49,6 +49,13 @@ for full details, see :ref:`running-mypy`. Asks mypy to type check the provided string as a program. +.. option:: --ignore-path + + Asks mypy to ignore a given file name, directory name or subpath while + recursively discovering files to check. This flag may be repeated multiple + times. + + Optional arguments ****************** diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index b747b31136cd2..3c293eab7d91b 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -197,6 +197,15 @@ section of the command line docs. This option may only be set in the global section (``[mypy]``). +.. confval:: ignore_path + + :type: comma-separated list of strings + + A comma-separated list of file names, directory names or subpaths which mypy + should ignore while recursively discovering files to check. + + This option may only be set in the global section (``[mypy]``). + .. confval:: namespace_packages :type: boolean diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst index 3498aaf072753..e2c3230a03f7b 100644 --- a/docs/source/running_mypy.rst +++ b/docs/source/running_mypy.rst @@ -390,7 +390,8 @@ to modules to type check. - Mypy will check all paths provided that correspond to files. - Mypy will recursively discover and check all files ending in ``.py`` or - ``.pyi`` in directory paths provided. + ``.pyi`` in directory paths provided, after accounting for + :option:`--ignore-path `. - For each file to be checked, mypy will attempt to associate the file (e.g. ``project/foo/bar/baz.py``) with a fully qualified module name (e.g. diff --git a/mypy/build.py b/mypy/build.py index 0ea4f643d20b5..ae4653828bc03 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -2769,14 +2769,12 @@ def load_graph(sources: List[BuildSource], manager: BuildManager, "Duplicate module named '%s' (also at '%s')" % (st.id, graph[st.id].xpath), blocker=True, ) - p1 = len(pathlib.PurePath(st.xpath).parents) - p2 = len(pathlib.PurePath(graph[st.id].xpath).parents) - - if p1 != p2: - manager.errors.report( - -1, -1, - "Are you missing an __init__.py?" - ) + manager.errors.report( + -1, -1, + "Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid " + "checking one of them.", + severity='note' + ) manager.errors.raise_error() graph[st.id] = st diff --git a/mypy/config_parser.py b/mypy/config_parser.py index dd79869030e54..3e7bec4f64fac 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -83,6 +83,7 @@ def check_follow_imports(choice: str) -> str: 'custom_typing_module': str, 'custom_typeshed_dir': expand_path, 'mypy_path': lambda s: [expand_path(p.strip()) for p in re.split('[,:]', s)], + 'ignore_path': lambda s: [expand_path(p.strip()) for p in s.split(',')], 'files': split_and_match_files, 'quickstart_file': expand_path, 'junit_xml': expand_path, diff --git a/mypy/find_sources.py b/mypy/find_sources.py index 47d686cddcbc3..7907c25ef4376 100644 --- a/mypy/find_sources.py +++ b/mypy/find_sources.py @@ -91,6 +91,7 @@ def __init__(self, fscache: FileSystemCache, options: Options) -> None: self.fscache = fscache self.explicit_package_bases = get_explicit_package_bases(options) self.namespace_packages = options.namespace_packages + self.ignore_path = options.ignore_path def is_explicit_package_base(self, path: str) -> bool: assert self.explicit_package_bases @@ -103,9 +104,15 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]: names = sorted(self.fscache.listdir(path), key=keyfunc) for name in names: # Skip certain names altogether - if name == '__pycache__' or name.startswith('.') or name.endswith('~'): + if ( + name in ("__pycache__", "site-packages", "node_modules") + or name.startswith(".") + or name.endswith("~") + ): continue subpath = os.path.join(path, name) + if any(subpath.endswith(pattern.rstrip("/")) for pattern in self.ignore_path): + continue if self.fscache.isdir(subpath): sub_sources = self.find_sources_in_dir(subpath) diff --git a/mypy/main.py b/mypy/main.py index be2fab1bb0f9c..d6e226ba968e2 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -808,6 +808,13 @@ def add_invertible_flag(flag: str, code_group.add_argument( '--explicit-package-bases', action='store_true', help="Use current directory and MYPYPATH to determine module names of files passed") + code_group.add_argument( + "--ignore-path", + metavar="PATH", + action="append", + default=[], + help="File names, directory names or subpaths to avoid checking", + ) code_group.add_argument( '-m', '--module', action='append', metavar='MODULE', default=[], diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index d223058c03674..34033ae2fa9fd 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -443,9 +443,17 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]: names = sorted(self.fscache.listdir(package_path)) for name in names: # Skip certain names altogether - if name == '__pycache__' or name.startswith('.') or name.endswith('~'): + if ( + name in ("__pycache__", "site-packages", "node_modules") + or name.startswith(".") + or name.endswith("~") + ): continue subpath = os.path.join(package_path, name) + if self.options and any( + subpath.endswith(pattern.rstrip("/")) for pattern in self.options.ignore_path + ): + continue if self.fscache.isdir(subpath): # Only recurse into packages diff --git a/mypy/options.py b/mypy/options.py index 2a0b3e1114424..953335f8536e5 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -97,6 +97,8 @@ def __init__(self) -> None: # sufficient to determine module names for files. As a possible alternative, add a single # top-level __init__.py to your packages. self.explicit_package_bases = False + # File names, directory names or subpaths to avoid checking + self.ignore_path = [] # type: List[str] # disallow_any options self.disallow_any_generics = False diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py index 5cedec338bbcd..ada14d465079f 100644 --- a/mypy/test/test_find_sources.py +++ b/mypy/test/test_find_sources.py @@ -252,3 +252,24 @@ def test_find_sources_namespace_multi_dir(self) -> None: finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options) assert find_sources(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")] + + def test_find_sources_ignore_path(self) -> None: + options = Options() + + finder = SourceFinder(FakeFSCache({"/dir/a.py", "/dir/venv/site-packages/b.py"}), options) + assert find_sources(finder, "/") == [("a", "/dir")] + + options.ignore_path = ["/pkg/a1"] + files = { + "/pkg/a1/b/c/d/e.py", + "/pkg/a1/b/f.py", + "/pkg/a2/__init__.py", + "/pkg/a2/b/c/d/e.py", + "/pkg/a2/b/f.py", + } + finder = SourceFinder(FakeFSCache(files), options) + assert find_sources(finder, "/") == [ + ("a2", "/pkg"), + ("e", "/pkg/a2/b/c/d"), + ("f", "/pkg/a2/b"), + ] diff --git a/mypy_self_check.ini b/mypy_self_check.ini index 2b7ed2b157c5c..15fea22c7dc20 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -19,3 +19,4 @@ pretty = True always_false = MYPYC plugins = misc/proper_plugin.py python_version = 3.5 +ignore_path = mypy/typeshed diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index c4d20ee78d614..87c3bd42b61f3 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -59,7 +59,7 @@ undef undef [out] dir/a.py: error: Duplicate module named 'a' (also at 'dir/subdir/a.py') -dir/a.py: error: Are you missing an __init__.py? +dir/a.py: note: Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid checking one of them. == Return code: 2 [case testCmdlineNonPackageSlash] @@ -125,19 +125,7 @@ mypy: can't decode file 'a.py': unknown encoding: uft-8 # type: ignore [out] two/mod/__init__.py: error: Duplicate module named 'mod' (also at 'one/mod/__init__.py') -== Return code: 2 - -[case promptsForgotInit] -# cmd: mypy a.py one/mod/a.py -[file one/__init__.py] -# type: ignore -[file a.py] -# type: ignore -[file one/mod/a.py] -#type: ignore -[out] -one/mod/a.py: error: Duplicate module named 'a' (also at 'a.py') -one/mod/a.py: error: Are you missing an __init__.py? +two/mod/__init__.py: note: Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid checking one of them. == Return code: 2 [case testFlagsFile]