From f59a8713df613a331202e4f02fbbc418a6f99628 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 3 Oct 2018 11:09:50 -0700 Subject: [PATCH] Namespace packages (PEP 420) (#5691) Tentative implementation of PEP 420. Fixes #1645. Clarification of the implementation: - `candidate_base_dirs` is a list of `(dir, verify)` tuples, laboriously pre-computed. - The old code just looped over this list, looking for `dir/`, and if found, it would verify that there were `__init__.py` files in all the right places (except if `verify` is false); the first success would be the hit; - In PEP 420 mode, if the above approach finds no hits, we do something different for those paths that failed due to `__init__` verification; essentially we narrow down the list of candidate paths by checking for `__init__` files from the top down. Hopefully the last test added clarifies this. --- mypy/modulefinder.py | 44 ++++++++++++++ test-data/unit/check-modules.test | 95 +++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index b9b79e3b4f09..3eb7dec89d83 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -170,6 +170,7 @@ def _find_module(self, id: str) -> Optional[str]: # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. seplast = os.sep + components[-1] # so e.g. '/baz' sepinit = os.sep + '__init__' + near_misses = [] # Collect near misses for namespace mode (see below). for base_dir, verify in candidate_base_dirs: base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' # Prefer package over module, i.e. baz/__init__.py* over baz.py*. @@ -178,10 +179,12 @@ def _find_module(self, id: str) -> Optional[str]: path_stubs = base_path + '-stubs' + sepinit + extension if fscache.isfile_case(path): if verify and not verify_module(fscache, id, path): + near_misses.append(path) continue return path elif fscache.isfile_case(path_stubs): if verify and not verify_module(fscache, id, path_stubs): + near_misses.append(path_stubs) continue return path_stubs # No package, look for module. @@ -189,8 +192,36 @@ def _find_module(self, id: str) -> Optional[str]: path = base_path + extension if fscache.isfile_case(path): if verify and not verify_module(fscache, id, path): + near_misses.append(path) continue return path + + # In namespace mode, re-check those entries that had 'verify'. + # Assume search path entries xxx, yyy and zzz, and we're + # looking for foo.bar.baz. Suppose near_misses has: + # + # - xxx/foo/bar/baz.py + # - yyy/foo/bar/baz/__init__.py + # - zzz/foo/bar/baz.pyi + # + # If any of the foo directories has __init__.py[i], it wins. + # Else, we look for foo/bar/__init__.py[i], etc. If there are + # none, the first hit wins. Note that this does not take into + # account whether the lowest-level module is a file (baz.py), + # a package (baz/__init__.py), or a stub file (baz.pyi) -- for + # these the first one encountered along the search path wins. + # + # The helper function highest_init_level() returns an int that + # indicates the highest level at which a __init__.py[i] file + # is found; if no __init__ was found it returns 0, if we find + # only foo/bar/__init__.py it returns 1, and if we have + # foo/__init__.py it returns 2 (regardless of what's in + # foo/bar). It doesn't look higher than that. + if self.options and self.options.namespace_packages and near_misses: + levels = [highest_init_level(fscache, id, path) for path in near_misses] + index = levels.index(max(levels)) + return near_misses[index] + return None def find_modules_recursive(self, module: str) -> List[BuildSource]: @@ -236,6 +267,19 @@ def verify_module(fscache: FileSystemCache, id: str, path: str) -> bool: return True +def highest_init_level(fscache: FileSystemCache, id: str, path: str) -> int: + """Compute the highest level where an __init__ file is found.""" + if path.endswith(('__init__.py', '__init__.pyi')): + path = os.path.dirname(path) + level = 0 + for i in range(id.count('.')): + path = os.path.dirname(path) + if any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension))) + for extension in PYTHON_EXTENSIONS): + level = i + 1 + return level + + def mypy_path() -> List[str]: path_env = os.getenv('MYPYPATH') if not path_env: diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index 36cd21b25189..23284c23bfb0 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -1,4 +1,5 @@ -- Type checker test cases dealing with modules and imports. +-- Towards the end there are tests for PEP 420 (namespace packages, i.e. __init__.py-less packages). [case testAccessImportedDefinitions] import m @@ -2525,3 +2526,97 @@ def __radd__(self) -> int: ... [case testFunctionWithInPlaceDunderName] def __iadd__(self) -> int: ... + +-- Tests for PEP 420 namespace packages. + +[case testClassicPackage] +from foo.bar import x +[file foo/__init__.py] +# empty +[file foo/bar.py] +x = 0 + +[case testClassicNotPackage] +from foo.bar import x +[file foo/bar.py] +x = 0 +[out] +main:1: error: Cannot find module named 'foo.bar' +main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) + +[case testNamespacePackage] +# flags: --namespace-packages +from foo.bar import x +reveal_type(x) # E: Revealed type is 'builtins.int' +[file foo/bar.py] +x = 0 + +[case testNamespacePackageWithMypyPath] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bax import x +from foo.bay import y +from foo.baz import z +reveal_type(x) # E: Revealed type is 'builtins.int' +reveal_type(y) # E: Revealed type is 'builtins.int' +reveal_type(z) # E: Revealed type is 'builtins.int' +[file xx/foo/bax.py] +x = 0 +[file yy/foo/bay.py] +y = 0 +[file foo/baz.py] +z = 0 +[file mypy.ini] +[[mypy] +mypy_path = tmp/xx, tmp/yy + +[case testClassicPackageIgnoresEarlierNamespacePackage] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bar import y +reveal_type(y) # E: Revealed type is 'builtins.int' +[file xx/foo/bar.py] +x = '' +[file yy/foo/bar.py] +y = 0 +[file yy/foo/__init__.py] +[file mypy.ini] +[[mypy] +mypy_path = tmp/xx, tmp/yy + +[case testNamespacePackagePickFirstOnMypyPath] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bar import x +reveal_type(x) # E: Revealed type is 'builtins.int' +[file xx/foo/bar.py] +x = 0 +[file yy/foo/bar.py] +x = '' +[file mypy.ini] +[[mypy] +mypy_path = tmp/xx, tmp/yy + +[case testNamespacePackageInsideClassicPackage] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bar.baz import x +reveal_type(x) # E: Revealed type is 'builtins.int' +[file xx/foo/bar/baz.py] +x = '' +[file yy/foo/bar/baz.py] +x = 0 +[file yy/foo/__init__.py] +[file mypy.ini] +[[mypy] +mypy_path = tmp/xx, tmp/yy + +[case testClassicPackageInsideNamespacePackage] +# flags: --namespace-packages --config-file tmp/mypy.ini +from foo.bar.baz.boo import x +reveal_type(x) # E: Revealed type is 'builtins.int' +[file xx/foo/bar/baz/boo.py] +x = '' +[file xx/foo/bar/baz/__init__.py] +[file yy/foo/bar/baz/boo.py] +x = 0 +[file yy/foo/bar/__init__.py] +[file mypy.ini] +[[mypy] +mypy_path = tmp/xx, tmp/yy