Namespace implementation (python#1645)

mypy/main.py

@@ -323,6 +323,8 @@ def add_invertible_flag(flag: str,
     parser.add_argument('--show-traceback', '--tb', action='store_true',
                         help="show traceback on fatal error")
     parser.add_argument('--stats', action='store_true', dest='dump_type_stats', help="dump stats")
+    parser.add_argument('--namespace-packages', action='store_true', dest='namespace_packages',
+                        help='Allow implicit namespace packages (PEP420)')
     parser.add_argument('--inferstats', action='store_true', dest='dump_inference_stats',
                         help="dump type inference stats")
     parser.add_argument('--custom-typing', metavar='MODULE', dest='custom_typing_module',
@@ -508,7 +510,8 @@ def add_invertible_flag(flag: str,
                  .format(special_opts.package))
         options.build_type = BuildType.MODULE
         lib_path = [os.getcwd()] + build.mypy_path()
-        targets = build.find_modules_recursive(special_opts.package, lib_path)
+        mod_discovery = build.ModuleDiscovery(lib_path, options.namespace_packages)
+        targets = mod_discovery.find_modules_recursive(special_opts.package)
         if not targets:
             fail("Can't find package '{}'".format(special_opts.package))
         return targets, options

mypy/options.py

@@ -164,6 +164,9 @@ def __init__(self) -> None:
         # Use stub builtins fixtures to speed up tests
         self.use_builtins_fixtures = False
 
+        # Allow implicit namespace packages (PEP420)
+        self.namespace_packages = False
+
         # -- experimental options --
         self.shadow_file = None  # type: Optional[Tuple[str, str]]
         self.show_column_numbers = False  # type: bool

mypy/semanal.py

@@ -182,8 +182,6 @@ class SemanticAnalyzerPass2(NodeVisitor[None]):
     This is the second phase of semantic analysis.
     """
 
-    # Library search paths
-    lib_path = None  # type: List[str]
     # Module name space
     modules = None  # type: Dict[str, MypyFile]
     # Global name space for current module
@@ -229,13 +227,9 @@ class SemanticAnalyzerPass2(NodeVisitor[None]):
     def __init__(self,
                  modules: Dict[str, MypyFile],
                  missing_modules: Set[str],
-                 lib_path: List[str], errors: Errors,
+                 errors: Errors,
                  plugin: Plugin) -> None:
-        """Construct semantic analyzer.
-
-        Use lib_path to search for modules, and report analysis errors
-        using the Errors instance.
-        """
+        """Construct semantic analyzer."""
         self.locals = [None]
         self.imports = set()
         self.type = None
@@ -244,7 +238,6 @@ def __init__(self,
         self.function_stack = []
         self.block_depth = [0]
         self.loop_depth = 0
-        self.lib_path = lib_path
         self.errors = errors
         self.modules = modules
         self.msg = MessageBuilder(errors, modules)

mypy/stubgen.py

@@ -156,7 +156,8 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int],
             module_all = getattr(mod, '__all__', None)
     else:
         # Find module by going through search path.
-        module_path = mypy.build.find_module(module, ['.'] + search_path)
+        md = mypy.build.ModuleDiscovery(['.'] + search_path)
+        module_path = md.find_module(module)
         if not module_path:
             raise SystemExit(
                 "Can't find module '{}' (consider using --search-path)".format(module))

mypy/test/testcheck.py

@@ -76,6 +76,7 @@
     'check-incomplete-fixture.test',
     'check-custom-plugin.test',
     'check-default-plugin.test',
+    'check-namespaces.test',
 ]
 
 
@@ -320,7 +321,8 @@ def parse_module(self,
             module_names = m.group(1)
             out = []
             for module_name in module_names.split(' '):
-                path = build.find_module(module_name, [test_temp_dir])
+                md = build.ModuleDiscovery([test_temp_dir], namespaces_allowed=False)
+                path = md.find_module(module_name)
                 assert path is not None, "Can't find ad hoc case file"
                 with open(path) as f:
                     program_text = f.read()

mypy/test/testdmypy.py

@@ -253,7 +253,8 @@ def parse_module(self,
             module_names = m.group(1)
             out = []
             for module_name in module_names.split(' '):
-                path = build.find_module(module_name, [test_temp_dir])
+                md = build.ModuleDiscovery([test_temp_dir])
+                path = md.find_module(module_name)
                 assert path is not None, "Can't find ad hoc case file"
                 with open(path) as f:
                     program_text = f.read()

mypy/test/testgraph.py

@@ -3,7 +3,7 @@
 from typing import AbstractSet, Dict, Set, List
 
 from mypy.myunit import Suite, assert_equal
-from mypy.build import BuildManager, State, BuildSourceSet
+from mypy.build import BuildManager, State, BuildSourceSet, ModuleDiscovery
 from mypy.build import topsort, strongly_connected_components, sorted_components, order_ascc
 from mypy.version import __version__
 from mypy.options import Options
@@ -41,14 +41,14 @@ def _make_manager(self) -> BuildManager:
         options = Options()
         manager = BuildManager(
             data_dir='',
-            lib_path=[],
             ignore_prefix='',
             source_set=BuildSourceSet([]),
             reports=Reports('', {}),
             options=options,
             version_id=__version__,
             plugin=Plugin(options),
             errors=errors,
+            module_discovery=ModuleDiscovery([]),
         )
         return manager
 

mypy/test/testmodulediscovery.py

@@ -0,0 +1,141 @@
+import os
+
+from unittest import mock, TestCase
+from typing import List, Set
+
+from mypy.build import ModuleDiscovery, find_module_clear_caches
+from mypy.myunit import Suite, assert_equal
+
+
+class ModuleDiscoveryTestCase(Suite):
+    def set_up(self) -> None:
+        self.files = set()  # type: Set[str]
+
+        self._setup_mock_filesystem()
+
+    def tear_down(self) -> None:
+        self._teardown_mock_filesystem()
+        find_module_clear_caches()
+
+    def _list_dir(self, path: str) -> List[str]:
+        res = set()
+
+        if not path.endswith(os.path.sep):
+            path = path + os.path.sep
+
+        for item in self.files:
+            if item.startswith(path):
+                remnant = item.replace(path, '')
+                segments = remnant.split(os.path.sep)
+                if segments:
+                    res.add(segments[0])
+
+        return list(res)
+
+    def _is_file(self, path: str) -> bool:
+        return path in self.files
+
+    def _is_dir(self, path: str) -> bool:
+        for item in self.files:
+            if not item.endswith('/'):
+                item += '/'
+            if item.startswith(path):
+                return True
+        return False
+
+    def _setup_mock_filesystem(self) -> None:
+        self._listdir_patcher = mock.patch('os.listdir', side_effect=self._list_dir)
+        self._listdir_mock = self._listdir_patcher.start()
+        self._isfile_patcher = mock.patch('os.path.isfile', side_effect=self._is_file)
+        self._isfile_mock = self._isfile_patcher.start()
+        self._isdir_patcher = mock.patch('os.path.isdir', side_effect=self._is_dir)
+        self._isdir_mock = self._isdir_patcher.start()
+
+    def _teardown_mock_filesystem(self) -> None:
+        self._listdir_patcher.stop()
+        self._isfile_patcher.stop()
+        self._isdir_patcher.stop()
+
+    def test_module_vs_package(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod.py'),
+            os.path.join('dir2', 'mod', '__init__.py'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=False)
+        path = m.find_module('mod')
+        assert_equal(path, os.path.join('dir1', 'mod.py'))
+
+        m = ModuleDiscovery(['dir2', 'dir1'], namespaces_allowed=False)
+        path = m.find_module('mod')
+        assert_equal(path, os.path.join('dir2', 'mod', '__init__.py'))
+
+    def test_package_in_different_directories(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', '__init__.py'),
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', '__init__.py'),
+            os.path.join('dir2', 'mod', 'b.py'),
+        }
+        m = ModuleDiscovery(['./dir1', './dir2'], namespaces_allowed=False)
+        path = m.find_module('mod.a')
+        assert_equal(path, os.path.join('dir1', 'mod', 'a.py'))
+
+        path = m.find_module('mod.b')
+        assert_equal(path, None)
+
+    def test_package_with_stubs(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', '__init__.py'),
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', '__init__.pyi'),
+            os.path.join('dir2', 'mod', 'b.pyi'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=False)
+        path = m.find_module('mod.a')
+        assert_equal(path, os.path.join('dir1', 'mod', 'a.py'))
+
+        path = m.find_module('mod.b')
+        assert_equal(path, os.path.join('dir2', 'mod', 'b.pyi'))
+
+    def test_namespaces(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', 'b.py'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=True)
+        path = m.find_module('mod.a')
+        assert_equal(path, os.path.join('dir1', 'mod', 'a.py'))
+
+        path = m.find_module('mod.b')
+        assert_equal(path, os.path.join('dir2', 'mod', 'b.py'))
+
+    def test_find_modules_recursive(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', '__init__.py'),
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', '__init__.pyi'),
+            os.path.join('dir2', 'mod', 'b.pyi'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=True)
+        srcs = m.find_modules_recursive('mod')
+        assert_equal([s.module for s in srcs], ['mod', 'mod.a', 'mod.b'])
+
+    def test_find_modules_recursive_with_namespace(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', 'b.py'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=True)
+        srcs = m.find_modules_recursive('mod')
+        assert_equal([s.module for s in srcs], ['mod.a', 'mod.b'])
+
+    def test_find_modules_recursive_with_stubs(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', '__init__.py'),
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', '__init__.pyi'),
+            os.path.join('dir2', 'mod', 'a.pyi'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=True)
+        srcs = m.find_modules_recursive('mod')
+        assert_equal([s.module for s in srcs], ['mod', 'mod.a'])

runtests.py

@@ -232,6 +232,7 @@ def test_path(*names: str):
     'testsolve',
     'testsubtypes',
     'testtypes',
+    'testmodulediscovery',
 )
 
 for f in find_files('mypy', prefix='test', suffix='.py'):

test-data/unit/check-namespaces.test

@@ -0,0 +1,87 @@
+-- Type checker test cases dealing with namespaces imports
+
+[case testAccessModuleInsideNamespace]
+# flags: --namespace-packages
+from ns import a
+[file ns/a.py]
+class A: pass
+def f(a: A) -> None: pass
+
+[case testAccessModuleInsideNamespaceNoNamespacePackages]
+from ns import a
+[file ns/a.py]
+class A: pass
+def f(a: A) -> None: pass
+[out]
+main:1: error: Cannot find module named 'ns'
+main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+
+[case testAccessPackageInsideNamespace]
+# flags: --namespace-packages
+from ns import a
+[file ns/a/__init__.py]
+class A: pass
+def f(a: A) -> None: pass
+
+[case testAccessPackageInsideNamespaceLocatedInSeparateDirectories]
+# flags: --config-file tmp/mypy.ini
+from ns import a, b
+[file mypy.ini]
+[[mypy]
+namespace_packages = True
+mypy_path = ./tmp/dir1:./tmp/dir2
+[file dir1/ns/a/__init__.py]
+class A: pass
+def f(a: A) -> None: pass
+[file dir2/ns/b.py]
+class B: pass
+def f(a: B) -> None: pass
+
+[case testConflictingPackageAndNamespaceFromImport]
+# flags: --config-file tmp/mypy.ini
+from pkg_or_ns import a
+from pkg_or_ns import b # E: Module 'pkg_or_ns' has no attribute 'b'
+[file mypy.ini]
+[[mypy]
+namespace_packages = True
+mypy_path = ./tmp/dir:./tmp/other_dir
+[file dir/pkg_or_ns/__init__.py]
+[file dir/pkg_or_ns/a.py]
+[file other_dir/pkg_or_ns/b.py]
+
+[case testConflictingPackageAndNamespaceImport]
+# flags: --config-file tmp/mypy.ini
+import pkg_or_ns.a
+import pkg_or_ns.b
+[file mypy.ini]
+[[mypy]
+namespace_packages = True
+mypy_path = ./tmp/dir:./tmp/other_dir
+[file dir/pkg_or_ns/__init__.py]
+[file dir/pkg_or_ns/a.py]
+[file other_dir/pkg_or_ns/b.py]
+[out]
+main:3: error: Cannot find module named 'pkg_or_ns.b'
+main:3: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+
+[case testConflictingModuleAndNamespace]
+# flags: --config-file tmp/mypy.ini
+from mod_or_ns import a
+from mod_or_ns import b # E: Module 'mod_or_ns' has no attribute 'b'
+[file mypy.ini]
+[[mypy]
+namespace_packages = True
+mypy_path = ./tmp/dir:./tmp/other_dir
+[file dir/mod_or_ns.py]
+a = None
+[file other_dir/mod_or_ns/b.py]
+
+[case testeNamespaceInsidePackage]
+# flags: --config-file tmp/mypy.ini
+from pkg.ns import a
+[file mypy.ini]
+[[mypy]
+namespace_packages = True
+[file pkg/__init__.py]
+[file pkg/ns/a.py]
+[out]