From 715b7686cc25ddd0088709ae33dbf841920ee2c4 Mon Sep 17 00:00:00 2001 From: falsedrow <140980541+falsedrow@users.noreply.github.com> Date: Fri, 25 Oct 2024 15:20:18 -0400 Subject: [PATCH] Filter to possible package paths before trying to resolve a module (#18038) With a long sys.path (it's got 300 entries), this removes 94% of stat syscalls from running mypy. With all the filesystem caching, that's only a small time savings, though it will depend on your filesystem. Local benchmarks showed a 20% time savings but they're pretty noisy from all the I/O. --------- Co-authored-by: Daniel Jacobowitz Co-authored-by: hauntsaninja --- mypy/modulefinder.py | 21 ++++++++++++--------- mypy/test/testmodulefinder.py | 2 +- test-data/unit/check-modules.test | 7 ++++++- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 49c39a9ce91c..3b6e1a874d37 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -331,8 +331,6 @@ def _find_module_non_stub_helper( # If this is not a directory then we can't traverse further into it if not self.fscache.isdir(dir_path): break - if approved_stub_package_exists(".".join(components)): - return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED if plausible_match: return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS else: @@ -414,9 +412,16 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: third_party_inline_dirs: PackageDirs = [] third_party_stubs_dirs: PackageDirs = [] found_possible_third_party_missing_type_hints = False - need_installed_stubs = False # Third-party stub/typed packages + candidate_package_dirs = { + package_dir[0] + for component in (components[0], components[0] + "-stubs") + for package_dir in self.find_lib_path_dirs(component, self.search_paths.package_path) + } for pkg_dir in self.search_paths.package_path: + pkg_dir = os.path.normpath(pkg_dir) + if pkg_dir not in candidate_package_dirs: + continue stub_name = components[0] + "-stubs" stub_dir = os_path_join(pkg_dir, stub_name) if fscache.isdir(stub_dir): @@ -445,11 +450,10 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: if isinstance(non_stub_match, ModuleNotFoundReason): if non_stub_match is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS: found_possible_third_party_missing_type_hints = True - elif non_stub_match is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED: - need_installed_stubs = True else: third_party_inline_dirs.append(non_stub_match) self._update_ns_ancestors(components, non_stub_match) + if self.options and self.options.use_builtins_fixtures: # Everything should be in fixtures. third_party_inline_dirs.clear() @@ -548,12 +552,11 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult: if ancestor is not None: return ancestor - if need_installed_stubs: + if approved_stub_package_exists(id): return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED - elif found_possible_third_party_missing_type_hints: + if found_possible_third_party_missing_type_hints: return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS - else: - return ModuleNotFoundReason.NOT_FOUND + return ModuleNotFoundReason.NOT_FOUND def find_modules_recursive(self, module: str) -> list[BuildSource]: module_path = self.find_module(module, fast_path=True) diff --git a/mypy/test/testmodulefinder.py b/mypy/test/testmodulefinder.py index 943913d6cadb..bec0e60c551f 100644 --- a/mypy/test/testmodulefinder.py +++ b/mypy/test/testmodulefinder.py @@ -165,7 +165,7 @@ def setUp(self) -> None: self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options) def path(self, *parts: str) -> str: - return os.path.join(self.package_dir, *parts) + return os.path.normpath(os.path.join(self.package_dir, *parts)) def test__packages_with_ns(self) -> None: cases = [ diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index 5fd48577e436..f368f244eb34 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -3146,8 +3146,13 @@ main:1: note: (or run "mypy --install-types" to install all missing stub package main:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports main:2: error: Library stubs not installed for "bleach.abc" -[case testMissingSubmoduleOfInstalledStubPackageIgnored] +[case testMissingSubmoduleOfInstalledStubPackageIgnored-xfail] # flags: --ignore-missing-imports + +# TODO: testMissingSubmoduleOfInstalledStubPackageIgnored was regressed in +# https://github.com/python/mypy/pull/15347 but didn't cause failures because we don't have a +# package path in this unit test + import bleach.xyz from bleach.abc import fgh [file bleach/__init__.pyi]