Add --ignore-path

Resolves python#4675, resolves python#9981

Additionally, we always ignore site-packages and node_modules.
Also note that this doesn't really affect import discovery; it only
directly affects passing files or packages to mypy.

The additional check before suggesting "are you missing an
__init__.py" didn't make any sense to me, so I removed it, appended to
the message and downgraded the severity to note.

docs/source/command_line.rst

@@ -49,6 +49,13 @@ for full details, see :ref:`running-mypy`.
     Asks mypy to type check the provided string as a program.
 
 
+.. option:: --ignore-path
+
+    Asks mypy to ignore a given file name, directory name or subpath while
+    recursively discovering files to check. This flag may be repeated multiple
+    times.
+
+
 Optional arguments
 ******************
 

docs/source/config_file.rst

@@ -197,6 +197,15 @@ section of the command line docs.
 
     This option may only be set in the global section (``[mypy]``).
 
+.. confval:: ignore_path
+
+    :type: comma-separated list of strings
+
+    A comma-separated list of file names, directory names or subpaths which mypy
+    should ignore while recursively discovering files to check.
+
+    This option may only be set in the global section (``[mypy]``).
+
 .. confval:: namespace_packages
 
     :type: boolean

docs/source/running_mypy.rst

@@ -390,7 +390,8 @@ to modules to type check.
 - Mypy will check all paths provided that correspond to files.
 
 - Mypy will recursively discover and check all files ending in ``.py`` or
-  ``.pyi`` in directory paths provided.
+  ``.pyi`` in directory paths provided, after accounting for
+  :option:`--ignore-path <mypy --ignore-path>`.
 
 - For each file to be checked, mypy will attempt to associate the file (e.g.
   ``project/foo/bar/baz.py``) with a fully qualified module name (e.g.

mypy/build.py

@@ -2769,14 +2769,12 @@ def load_graph(sources: List[BuildSource], manager: BuildManager,
                 "Duplicate module named '%s' (also at '%s')" % (st.id, graph[st.id].xpath),
                 blocker=True,
             )
-            p1 = len(pathlib.PurePath(st.xpath).parents)
-            p2 = len(pathlib.PurePath(graph[st.id].xpath).parents)
-
-            if p1 != p2:
-                manager.errors.report(
-                    -1, -1,
-                    "Are you missing an __init__.py?"
-                )
+            manager.errors.report(
+                -1, -1,
+                "Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid "
+                "checking one of them.",
+                severity='note'
+            )
 
             manager.errors.raise_error()
         graph[st.id] = st

mypy/config_parser.py

@@ -83,6 +83,7 @@ def check_follow_imports(choice: str) -> str:
     'custom_typing_module': str,
     'custom_typeshed_dir': expand_path,
     'mypy_path': lambda s: [expand_path(p.strip()) for p in re.split('[,:]', s)],
+    'ignore_path': lambda s: [expand_path(p.strip()) for p in s.split(',')],
     'files': split_and_match_files,
     'quickstart_file': expand_path,
     'junit_xml': expand_path,

mypy/find_sources.py

@@ -91,6 +91,7 @@ def __init__(self, fscache: FileSystemCache, options: Options) -> None:
         self.fscache = fscache
         self.explicit_package_bases = get_explicit_package_bases(options)
         self.namespace_packages = options.namespace_packages
+        self.ignore_path = options.ignore_path
 
     def is_explicit_package_base(self, path: str) -> bool:
         assert self.explicit_package_bases
@@ -103,9 +104,15 @@ def find_sources_in_dir(self, path: str) -> List[BuildSource]:
         names = sorted(self.fscache.listdir(path), key=keyfunc)
         for name in names:
             # Skip certain names altogether
-            if name == '__pycache__' or name.startswith('.') or name.endswith('~'):
+            if (
+                name in ("__pycache__", "site-packages", "node_modules")
+                or name.startswith(".")
+                or name.endswith("~")
+            ):
                 continue
             subpath = os.path.join(path, name)
+            if any(subpath.endswith(pattern.rstrip("/")) for pattern in self.ignore_path):
+                continue
 
             if self.fscache.isdir(subpath):
                 sub_sources = self.find_sources_in_dir(subpath)

mypy/main.py

@@ -808,6 +808,13 @@ def add_invertible_flag(flag: str,
     code_group.add_argument(
         '--explicit-package-bases', action='store_true',
         help="Use current directory and MYPYPATH to determine module names of files passed")
+    code_group.add_argument(
+        "--ignore-path",
+        metavar="PATH",
+        action="append",
+        default=[],
+        help="File names, directory names or subpaths to avoid checking",
+    )
     code_group.add_argument(
         '-m', '--module', action='append', metavar='MODULE',
         default=[],

mypy/modulefinder.py

@@ -443,9 +443,17 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]:
         names = sorted(self.fscache.listdir(package_path))
         for name in names:
             # Skip certain names altogether
-            if name == '__pycache__' or name.startswith('.') or name.endswith('~'):
+            if (
+                name in ("__pycache__", "site-packages", "node_modules")
+                or name.startswith(".")
+                or name.endswith("~")
+            ):
                 continue
             subpath = os.path.join(package_path, name)
+            if self.options and any(
+                subpath.endswith(pattern.rstrip("/")) for pattern in self.options.ignore_path
+            ):
+                continue
 
             if self.fscache.isdir(subpath):
                 # Only recurse into packages

mypy/options.py

@@ -97,6 +97,8 @@ def __init__(self) -> None:
         # sufficient to determine module names for files. As a possible alternative, add a single
         # top-level __init__.py to your packages.
         self.explicit_package_bases = False
+        # File names, directory names or subpaths to avoid checking
+        self.ignore_path = []  # type: List[str]
 
         # disallow_any options
         self.disallow_any_generics = False

mypy/test/test_find_sources.py

@@ -252,3 +252,34 @@ def test_find_sources_namespace_multi_dir(self) -> None:
 
         finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options)
         assert find_sources(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")]
+
+    def test_find_sources_ignore_path(self) -> None:
+        options = Options()
+        options.namespace_packages = True
+
+        finder = SourceFinder(FakeFSCache({"/dir/a.py", "/dir/venv/site-packages/b.py"}), options)
+        assert find_sources(finder, "/") == [("a", "/dir")]
+
+        files = {
+            "/pkg/a1/b/c/d/e.py",
+            "/pkg/a1/b/f.py",
+            "/pkg/a2/__init__.py",
+            "/pkg/a2/b/c/d/e.py",
+            "/pkg/a2/b/f.py",
+        }
+
+        options.ignore_path = ["/pkg/a1"]
+        finder = SourceFinder(FakeFSCache(files), options)
+        assert find_sources(finder, "/") == [
+            ("a2", "/pkg"),
+            ("a2.b.c.d.e", "/pkg"),
+            ("a2.b.f", "/pkg"),
+        ]
+
+        options.ignore_path = ["f.py"]
+        finder = SourceFinder(FakeFSCache(files), options)
+        assert find_sources(finder, "/") == [
+            ("a2", "/pkg"),
+            ("a2.b.c.d.e", "/pkg"),
+            ("e", "/pkg/a1/b/c/d"),
+        ]

mypy_self_check.ini

@@ -19,3 +19,4 @@ pretty = True
 always_false = MYPYC
 plugins = misc/proper_plugin.py
 python_version = 3.5
+ignore_path = mypy/typeshed

test-data/unit/cmdline.test

@@ -59,7 +59,7 @@ undef
 undef
 [out]
 dir/a.py: error: Duplicate module named 'a' (also at 'dir/subdir/a.py')
-dir/a.py: error: Are you missing an __init__.py?
+dir/a.py: note: Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid checking one of them.
 == Return code: 2
 
 [case testCmdlineNonPackageSlash]
@@ -125,19 +125,7 @@ mypy: can't decode file 'a.py': unknown encoding: uft-8
 # type: ignore
 [out]
 two/mod/__init__.py: error: Duplicate module named 'mod' (also at 'one/mod/__init__.py')
-== Return code: 2
-
-[case promptsForgotInit]
-# cmd: mypy a.py one/mod/a.py
-[file one/__init__.py]
-# type: ignore
-[file a.py]
-# type: ignore
-[file one/mod/a.py]
-#type: ignore
-[out]
-one/mod/a.py: error: Duplicate module named 'a' (also at 'a.py')
-one/mod/a.py: error: Are you missing an __init__.py?
+two/mod/__init__.py: note: Are you missing an __init__.py? Alternatively, consider using --ignore-path to avoid checking one of them.
 == Return code: 2
 
 [case testFlagsFile]