diff --git a/astroid/inference.py b/astroid/inference.py index cd66a91709..8d85664b34 100644 --- a/astroid/inference.py +++ b/astroid/inference.py @@ -23,8 +23,6 @@ Union, ) -import wrapt - from astroid import bases, decorators, helpers, nodes, protocols, util from astroid.context import ( CallContext, @@ -1037,28 +1035,6 @@ def infer_ifexp(self, context=None): nodes.IfExp._infer = infer_ifexp # type: ignore[assignment] -# pylint: disable=dangerous-default-value -@wrapt.decorator -def _cached_generator( - func, instance: _FunctionDefT, args, kwargs, _cache={} # noqa: B006 -): - node = instance - try: - return iter(_cache[func, id(node)]) - except KeyError: - result = func(*args, **kwargs) - # Need to keep an iterator around - original, copy = itertools.tee(result) - _cache[func, id(node)] = list(copy) - return original - - -# When inferring a property, we instantiate a new `objects.Property` object, -# which in turn, because it inherits from `FunctionDef`, sets itself in the locals -# of the wrapping frame. This means that every time we infer a property, the locals -# are mutated with a new instance of the property. This is why we cache the result -# of the function's inference. -@_cached_generator def infer_functiondef( self: _FunctionDefT, context: Optional[InferenceContext] = None ) -> Generator[Union["Property", _FunctionDefT], None, InferenceErrorInfo]: @@ -1066,13 +1042,25 @@ def infer_functiondef( yield self return InferenceErrorInfo(node=self, context=context) + # When inferring a property, we instantiate a new `objects.Property` object, + # which in turn, because it inherits from `FunctionDef`, sets itself in the locals + # of the wrapping frame. This means that every time we infer a property, the locals + # are mutated with a new instance of the property. To avoid this, we detect this + # scenario and avoid passing the `parent` argument to the constructor. + parent_frame = self.parent.frame(future=True) + property_already_in_parent_locals = self.name in parent_frame.locals and any( + isinstance(val, objects.Property) for val in parent_frame.locals[self.name] + ) + prop_func = objects.Property( function=self, name=self.name, lineno=self.lineno, - parent=self.parent, + parent=self.parent if not property_already_in_parent_locals else None, col_offset=self.col_offset, ) + if property_already_in_parent_locals: + prop_func.parent = self.parent prop_func.postinit(body=[], args=self.args, doc_node=self.doc_node) yield prop_func return InferenceErrorInfo(node=self, context=context) diff --git a/astroid/interpreter/objectmodel.py b/astroid/interpreter/objectmodel.py index cf9227b510..4fd79596b7 100644 --- a/astroid/interpreter/objectmodel.py +++ b/astroid/interpreter/objectmodel.py @@ -25,6 +25,7 @@ import os import pprint import types +from functools import lru_cache from typing import TYPE_CHECKING, List, Optional import astroid @@ -100,6 +101,7 @@ def __get__(self, instance, cls=None): def __contains__(self, name): return name in self.attributes() + @lru_cache() # noqa def attributes(self) -> List[str]: """Get the attributes which are exported by this object model.""" return [o[LEN_OF_IMPL_PREFIX:] for o in dir(self) if o.startswith(IMPL_PREFIX)] diff --git a/astroid/manager.py b/astroid/manager.py index 9651bfada9..653c1390a2 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -18,6 +18,7 @@ from astroid.interpreter._import import spec from astroid.modutils import ( NoSourceFile, + _cache_normalize_path_, file_info_from_modpath, get_source_file, is_module_name_part_of_extension_package_whitelist, @@ -365,8 +366,24 @@ def bootstrap(self): def clear_cache(self) -> None: """Clear the underlying cache, bootstrap the builtins module and re-register transforms.""" + # import here because of cyclic imports + # pylint: disable=import-outside-toplevel + from astroid.inference_tip import clear_inference_tip_cache + from astroid.interpreter.objectmodel import ObjectModel + from astroid.nodes.node_classes import LookupMixIn + + clear_inference_tip_cache() + self.astroid_cache.clear() AstroidManager.brain["_transform"] = TransformVisitor() + + for lru_cache in ( + LookupMixIn.lookup, + _cache_normalize_path_, + ObjectModel.attributes, + ): + lru_cache.cache_clear() + self.bootstrap() # Reload brain plugins. During initialisation this is done in astroid.__init__.py diff --git a/astroid/modutils.py b/astroid/modutils.py index cfb3f85184..06fda983d7 100644 --- a/astroid/modutils.py +++ b/astroid/modutils.py @@ -22,8 +22,9 @@ import sys import sysconfig import types +from functools import lru_cache from pathlib import Path -from typing import Dict, Set +from typing import Set from astroid.const import IS_JYTHON, IS_PYPY from astroid.interpreter._import import spec, util @@ -138,7 +139,9 @@ def _handle_blacklist(blacklist, dirnames, filenames): filenames.remove(norecurs) -_NORM_PATH_CACHE: Dict[str, str] = {} +@lru_cache() +def _cache_normalize_path_(path: str) -> str: + return _normalize_path(path) def _cache_normalize_path(path: str) -> str: @@ -146,13 +149,9 @@ def _cache_normalize_path(path: str) -> str: # _module_file calls abspath on every path in sys.path every time it's # called; on a larger codebase this easily adds up to half a second just # assembling path components. This cache alleviates that. - try: - return _NORM_PATH_CACHE[path] - except KeyError: - if not path: # don't cache result for '' - return _normalize_path(path) - result = _NORM_PATH_CACHE[path] = _normalize_path(path) - return result + if not path: # don't cache result for '' + return _normalize_path(path) + return _cache_normalize_path_(path) def load_module_from_name(dotted_name: str) -> types.ModuleType: diff --git a/astroid/nodes/node_classes.py b/astroid/nodes/node_classes.py index ca2d217c95..c1cd4f886f 100644 --- a/astroid/nodes/node_classes.py +++ b/astroid/nodes/node_classes.py @@ -367,7 +367,7 @@ def get_children(self): class LookupMixIn: """Mixin to look up a name in the right scope.""" - @lru_cache(maxsize=None) # pylint: disable=cache-max-size-none # noqa + @lru_cache() # noqa def lookup(self, name: str) -> typing.Tuple[str, typing.List[NodeNG]]: """Lookup where the given variable is assigned. diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 6f0a7b16f4..96239233b3 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -16,7 +16,9 @@ from astroid import manager, test_utils from astroid.const import IS_JYTHON from astroid.exceptions import AstroidBuildingError, AstroidImportError +from astroid.modutils import is_standard_module from astroid.nodes import Const +from astroid.nodes.scoped_nodes import ClassDef from . import resources @@ -317,6 +319,43 @@ def test_borg(self) -> None: class ClearCacheTest(unittest.TestCase, resources.AstroidCacheSetupMixin): + def test_clear_cache_clears_other_lru_caches(self) -> None: + lrus = ( + astroid.nodes.node_classes.LookupMixIn.lookup, + astroid.modutils._cache_normalize_path_, + astroid.interpreter.objectmodel.ObjectModel.attributes, + ) + + # Get a baseline for the size of the cache after simply calling bootstrap() + baseline_cache_infos = [lru.cache_info() for lru in lrus] + + # Generate some hits and misses + ClassDef().lookup("garbage") + is_standard_module("unittest", std_path=["garbage_path"]) + astroid.interpreter.objectmodel.ObjectModel().attributes() + + # Did the hits or misses actually happen? + incremented_cache_infos = [lru.cache_info() for lru in lrus] + for incremented_cache, baseline_cache in zip( + incremented_cache_infos, baseline_cache_infos + ): + with self.subTest(incremented_cache=incremented_cache): + self.assertGreater( + incremented_cache.hits + incremented_cache.misses, + baseline_cache.hits + baseline_cache.misses, + ) + + astroid.MANAGER.clear_cache() # also calls bootstrap() + + # The cache sizes are now as low or lower than the original baseline + cleared_cache_infos = [lru.cache_info() for lru in lrus] + for cleared_cache, baseline_cache in zip( + cleared_cache_infos, baseline_cache_infos + ): + with self.subTest(cleared_cache=cleared_cache): + # less equal because the "baseline" might have had multiple calls to bootstrap() + self.assertLessEqual(cleared_cache.currsize, baseline_cache.currsize) + def test_brain_plugins_reloaded_after_clearing_cache(self) -> None: astroid.MANAGER.clear_cache() format_call = astroid.extract_node("''.format()") diff --git a/tests/unittest_modutils.py b/tests/unittest_modutils.py index 3fb92a845b..3d15fb632c 100644 --- a/tests/unittest_modutils.py +++ b/tests/unittest_modutils.py @@ -179,7 +179,7 @@ def test_load_packages_without_init(self) -> None: https://github.com/PyCQA/astroid/issues/1327 """ tmp_dir = Path(tempfile.gettempdir()) - self.addCleanup(os.chdir, os.curdir) + self.addCleanup(os.chdir, os.getcwd()) os.chdir(tmp_dir) self.addCleanup(shutil.rmtree, tmp_dir / "src") @@ -288,6 +288,8 @@ def test_custom_path(self) -> None: self.assertTrue( modutils.is_standard_module("data.module", (os.path.abspath(datadir),)) ) + # "" will evaluate to cwd + self.assertTrue(modutils.is_standard_module("data.module", ("",))) def test_failing_edge_cases(self) -> None: # using a subpackage/submodule path as std_path argument