From 35171c446a8fa4cfccd7cf6a96a841c3afee4bb8 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 20 Apr 2022 09:30:58 -0400 Subject: [PATCH 01/19] Proof of concept for extending --- astroid/inference.py | 37 ++++++++++-------------------- astroid/interpreter/objectmodel.py | 2 ++ astroid/manager.py | 24 ++++++++++++++++++- astroid/modutils.py | 10 +++----- astroid/nodes/node_classes.py | 2 +- 5 files changed, 41 insertions(+), 34 deletions(-) diff --git a/astroid/inference.py b/astroid/inference.py index cd66a91709..fa65250380 100644 --- a/astroid/inference.py +++ b/astroid/inference.py @@ -23,8 +23,6 @@ Union, ) -import wrapt - from astroid import bases, decorators, helpers, nodes, protocols, util from astroid.context import ( CallContext, @@ -1037,28 +1035,6 @@ def infer_ifexp(self, context=None): nodes.IfExp._infer = infer_ifexp # type: ignore[assignment] -# pylint: disable=dangerous-default-value -@wrapt.decorator -def _cached_generator( - func, instance: _FunctionDefT, args, kwargs, _cache={} # noqa: B006 -): - node = instance - try: - return iter(_cache[func, id(node)]) - except KeyError: - result = func(*args, **kwargs) - # Need to keep an iterator around - original, copy = itertools.tee(result) - _cache[func, id(node)] = list(copy) - return original - - -# When inferring a property, we instantiate a new `objects.Property` object, -# which in turn, because it inherits from `FunctionDef`, sets itself in the locals -# of the wrapping frame. This means that every time we infer a property, the locals -# are mutated with a new instance of the property. This is why we cache the result -# of the function's inference. -@_cached_generator def infer_functiondef( self: _FunctionDefT, context: Optional[InferenceContext] = None ) -> Generator[Union["Property", _FunctionDefT], None, InferenceErrorInfo]: @@ -1066,13 +1042,24 @@ def infer_functiondef( yield self return InferenceErrorInfo(node=self, context=context) + # When inferring a property, we instantiate a new `objects.Property` object, + # which in turn, because it inherits from `FunctionDef`, sets itself in the locals + # of the wrapping frame. This means that every time we infer a property, the locals + # are mutated with a new instance of the property. To avoid this, we detect this + # scenario and avoid passing the `parent` argument to the constructor. + property_already_in_parent_locals = self.name in self.parent.locals and any( + isinstance(val, objects.Property) for val in self.parent.locals[self.name] + ) + prop_func = objects.Property( function=self, name=self.name, lineno=self.lineno, - parent=self.parent, + parent=self.parent if not property_already_in_parent_locals else None, col_offset=self.col_offset, ) + if property_already_in_parent_locals: + prop_func.parent = self.parent prop_func.postinit(body=[], args=self.args, doc_node=self.doc_node) yield prop_func return InferenceErrorInfo(node=self, context=context) diff --git a/astroid/interpreter/objectmodel.py b/astroid/interpreter/objectmodel.py index cf9227b510..7a112af13c 100644 --- a/astroid/interpreter/objectmodel.py +++ b/astroid/interpreter/objectmodel.py @@ -25,6 +25,7 @@ import os import pprint import types +from functools import lru_cache from typing import TYPE_CHECKING, List, Optional import astroid @@ -100,6 +101,7 @@ def __get__(self, instance, cls=None): def __contains__(self, name): return name in self.attributes() + @lru_cache() # pylint: disable=cache-max-size-none # noqa def attributes(self) -> List[str]: """Get the attributes which are exported by this object model.""" return [o[LEN_OF_IMPL_PREFIX:] for o in dir(self) if o.startswith(IMPL_PREFIX)] diff --git a/astroid/manager.py b/astroid/manager.py index 5bb3728f80..314f2a1a9b 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -16,6 +16,7 @@ from astroid.interpreter._import import spec from astroid.modutils import ( NoSourceFile, + _cache_normalize_path, file_info_from_modpath, get_source_file, is_module_name_part_of_extension_package_whitelist, @@ -59,6 +60,16 @@ class AstroidManager: max_inferable_values: ClassVar[int] = 100 def __init__(self): + # pylint: disable=import-outside-top-level + from astroid.interpreter.objectmodel import ObjectModel + from astroid.nodes.node_classes import LookupMixIn + + self._lru_caches = [ + LookupMixIn.lookup, + _cache_normalize_path, + ObjectModel.attributes, + ] + # NOTE: cache entries are added by the [re]builder self.astroid_cache = AstroidManager.brain["astroid_cache"] self._mod_file_cache = AstroidManager.brain["_mod_file_cache"] @@ -357,6 +368,17 @@ def bootstrap(self): raw_building._astroid_bootstrapping() def clear_cache(self): - """Clear the underlying cache. Also bootstraps the builtins module.""" + """Clear the underlying caches. Also bootstraps the builtins module.""" + from astroid.inference_tip import ( + clear_inference_tip_cache, + ) # pylint: disable=import-outside-toplevel + + clear_inference_tip_cache() + self.astroid_cache.clear() + + for lru_cache in self._lru_caches: + assert hasattr(lru_cache, "cache_clear"), lru_cache + lru_cache.cache_clear() + self.bootstrap() diff --git a/astroid/modutils.py b/astroid/modutils.py index 01135ef4c6..3f78dbf95b 100644 --- a/astroid/modutils.py +++ b/astroid/modutils.py @@ -22,6 +22,7 @@ import sys import sysconfig import types +from functools import lru_cache from pathlib import Path from typing import Dict, Set @@ -141,18 +142,13 @@ def _handle_blacklist(blacklist, dirnames, filenames): _NORM_PATH_CACHE: Dict[str, str] = {} +@lru_cache() def _cache_normalize_path(path: str) -> str: """Normalize path with caching.""" # _module_file calls abspath on every path in sys.path every time it's # called; on a larger codebase this easily adds up to half a second just # assembling path components. This cache alleviates that. - try: - return _NORM_PATH_CACHE[path] - except KeyError: - if not path: # don't cache result for '' - return _normalize_path(path) - result = _NORM_PATH_CACHE[path] = _normalize_path(path) - return result + return _normalize_path(path) def load_module_from_name(dotted_name: str) -> types.ModuleType: diff --git a/astroid/nodes/node_classes.py b/astroid/nodes/node_classes.py index f7f6231d5f..9055b2d3e0 100644 --- a/astroid/nodes/node_classes.py +++ b/astroid/nodes/node_classes.py @@ -366,7 +366,7 @@ def get_children(self): class LookupMixIn: """Mixin to look up a name in the right scope.""" - @lru_cache(maxsize=None) # pylint: disable=cache-max-size-none # noqa + @lru_cache() # pylint: disable=cache-max-size-none # noqa def lookup(self, name: str) -> typing.Tuple[str, typing.List[NodeNG]]: """Lookup where the given variable is assigned. From 0ea952c5069dc1fa7290c975269b1ecbf3f525fc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Apr 2022 13:54:06 +0000 Subject: [PATCH 02/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- astroid/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroid/manager.py b/astroid/manager.py index 314f2a1a9b..101d6de015 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -369,9 +369,9 @@ def bootstrap(self): def clear_cache(self): """Clear the underlying caches. Also bootstraps the builtins module.""" - from astroid.inference_tip import ( + from astroid.inference_tip import ( # pylint: disable=import-outside-toplevel clear_inference_tip_cache, - ) # pylint: disable=import-outside-toplevel + ) clear_inference_tip_cache() From f70ef40c4c286b9e63a7e1c1111dbfbf7f71da6d Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 20 Apr 2022 10:43:24 -0400 Subject: [PATCH 03/19] parent frame, not parent --- astroid/inference.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/astroid/inference.py b/astroid/inference.py index fa65250380..8d85664b34 100644 --- a/astroid/inference.py +++ b/astroid/inference.py @@ -1047,8 +1047,9 @@ def infer_functiondef( # of the wrapping frame. This means that every time we infer a property, the locals # are mutated with a new instance of the property. To avoid this, we detect this # scenario and avoid passing the `parent` argument to the constructor. - property_already_in_parent_locals = self.name in self.parent.locals and any( - isinstance(val, objects.Property) for val in self.parent.locals[self.name] + parent_frame = self.parent.frame(future=True) + property_already_in_parent_locals = self.name in parent_frame.locals and any( + isinstance(val, objects.Property) for val in parent_frame.locals[self.name] ) prop_func = objects.Property( From 035902579a07222f6ea121e442e039516ff117d3 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 20 Apr 2022 14:07:33 -0400 Subject: [PATCH 04/19] tinker with disables --- astroid/interpreter/objectmodel.py | 2 +- astroid/manager.py | 3 +-- astroid/nodes/node_classes.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/astroid/interpreter/objectmodel.py b/astroid/interpreter/objectmodel.py index 7a112af13c..4fd79596b7 100644 --- a/astroid/interpreter/objectmodel.py +++ b/astroid/interpreter/objectmodel.py @@ -101,7 +101,7 @@ def __get__(self, instance, cls=None): def __contains__(self, name): return name in self.attributes() - @lru_cache() # pylint: disable=cache-max-size-none # noqa + @lru_cache() # noqa def attributes(self) -> List[str]: """Get the attributes which are exported by this object model.""" return [o[LEN_OF_IMPL_PREFIX:] for o in dir(self) if o.startswith(IMPL_PREFIX)] diff --git a/astroid/manager.py b/astroid/manager.py index 101d6de015..e8b8b7f18b 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -59,8 +59,7 @@ class AstroidManager: } max_inferable_values: ClassVar[int] = 100 - def __init__(self): - # pylint: disable=import-outside-top-level + def __init__(self): # pylint: disable=import-outside-top-level from astroid.interpreter.objectmodel import ObjectModel from astroid.nodes.node_classes import LookupMixIn diff --git a/astroid/nodes/node_classes.py b/astroid/nodes/node_classes.py index 9055b2d3e0..8dce38b5cb 100644 --- a/astroid/nodes/node_classes.py +++ b/astroid/nodes/node_classes.py @@ -366,7 +366,7 @@ def get_children(self): class LookupMixIn: """Mixin to look up a name in the right scope.""" - @lru_cache() # pylint: disable=cache-max-size-none # noqa + @lru_cache() # noqa def lookup(self, name: str) -> typing.Tuple[str, typing.List[NodeNG]]: """Lookup where the given variable is assigned. From 0d4c0d3323b15a38ab663956ef2eb8d7978f7ec7 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 20 Apr 2022 15:57:11 -0400 Subject: [PATCH 05/19] Add unit test for clear_cache() --- astroid/manager.py | 4 ++-- tests/unittest_manager.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/astroid/manager.py b/astroid/manager.py index e8b8b7f18b..879fa12d89 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -10,6 +10,7 @@ import os import types import zipimport +from functools import _lru_cache_wrapper from typing import TYPE_CHECKING, ClassVar, List, Optional from astroid.exceptions import AstroidBuildingError, AstroidImportError @@ -63,7 +64,7 @@ def __init__(self): # pylint: disable=import-outside-top-level from astroid.interpreter.objectmodel import ObjectModel from astroid.nodes.node_classes import LookupMixIn - self._lru_caches = [ + self._lru_caches: List[_lru_cache_wrapper] = [ LookupMixIn.lookup, _cache_normalize_path, ObjectModel.attributes, @@ -377,7 +378,6 @@ def clear_cache(self): self.astroid_cache.clear() for lru_cache in self._lru_caches: - assert hasattr(lru_cache, "cache_clear"), lru_cache lru_cache.cache_clear() self.bootstrap() diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 4785975692..caebbb4f10 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -16,6 +16,7 @@ from astroid import manager, test_utils from astroid.const import IS_JYTHON from astroid.exceptions import AstroidBuildingError, AstroidImportError +from astroid.modutils import is_standard_module from . import resources @@ -315,5 +316,41 @@ def test_borg(self) -> None: self.assertIs(built, second_built) +class ClearCacheTest(unittest.TestCase): + def test_clear_cache(self) -> None: + # Get a baseline for the size of the cache after simply calling bootstrap() + baseline_cache_infos = [lru.cache_info() for lru in astroid.MANAGER._lru_caches] + + # Generate some hits and misses + # Exercise LookupMixIn.lookup() + astroid.nodes.scoped_nodes.ClassDef().lookup("garbage") + # Exercise _cache_normalize_path() + is_standard_module("unittest", std_path=["garbage_path"]) + # Exercise ObjectModel.attributes() + astroid.MANAGER.bootstrap() + + # Did the hits or misses actually happen? + for lru_cache, baseline_cache in zip( + astroid.MANAGER._lru_caches, baseline_cache_infos + ): + with self.subTest(lru_cache=lru_cache): + cache_info = lru_cache.cache_info() + self.assertTrue( + cache_info.hits > baseline_cache.hits + or cache_info.misses > baseline_cache.misses + ) + + astroid.MANAGER.clear_cache() # also calls bootstrap() + + # The cache sizes are now as low or lower than the original baseline + for lru_cache, baseline_cache in zip( + astroid.MANAGER._lru_caches, baseline_cache_infos + ): + with self.subTest(lru_cache=lru_cache): + cache_info = lru_cache.cache_info() + # less equal because the "baseline" might have had multiple calls to bootstrap() + self.assertLessEqual(cache_info.currsize, baseline_cache.currsize) + + if __name__ == "__main__": unittest.main() From 37b1d91317aa8f7a6a92cf6d10e16c6d7382707a Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 20 Apr 2022 16:45:08 -0400 Subject: [PATCH 06/19] Update test result --- astroid/modutils.py | 5 +---- tests/unittest_manager.py | 35 +++++++++++++++++----------------- tests/unittest_scoped_nodes.py | 2 -- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/astroid/modutils.py b/astroid/modutils.py index 3f78dbf95b..1c045218dc 100644 --- a/astroid/modutils.py +++ b/astroid/modutils.py @@ -24,7 +24,7 @@ import types from functools import lru_cache from pathlib import Path -from typing import Dict, Set +from typing import Set from astroid.const import IS_JYTHON, IS_PYPY from astroid.interpreter._import import spec, util @@ -139,9 +139,6 @@ def _handle_blacklist(blacklist, dirnames, filenames): filenames.remove(norecurs) -_NORM_PATH_CACHE: Dict[str, str] = {} - - @lru_cache() def _cache_normalize_path(path: str) -> str: """Normalize path with caching.""" diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index caebbb4f10..cfbe82d1eb 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -16,7 +16,9 @@ from astroid import manager, test_utils from astroid.const import IS_JYTHON from astroid.exceptions import AstroidBuildingError, AstroidImportError +from astroid.interpreter.objectmodel import ObjectModel from astroid.modutils import is_standard_module +from astroid.nodes.scoped_nodes import ClassDef from . import resources @@ -322,34 +324,33 @@ def test_clear_cache(self) -> None: baseline_cache_infos = [lru.cache_info() for lru in astroid.MANAGER._lru_caches] # Generate some hits and misses - # Exercise LookupMixIn.lookup() - astroid.nodes.scoped_nodes.ClassDef().lookup("garbage") - # Exercise _cache_normalize_path() + ClassDef().lookup("garbage") is_standard_module("unittest", std_path=["garbage_path"]) - # Exercise ObjectModel.attributes() - astroid.MANAGER.bootstrap() + ObjectModel().attributes() # Did the hits or misses actually happen? - for lru_cache, baseline_cache in zip( - astroid.MANAGER._lru_caches, baseline_cache_infos + incremented_cache_infos = [ + lru.cache_info() for lru in astroid.MANAGER._lru_caches + ] + for incremented_cache, baseline_cache in zip( + incremented_cache_infos, baseline_cache_infos ): - with self.subTest(lru_cache=lru_cache): - cache_info = lru_cache.cache_info() - self.assertTrue( - cache_info.hits > baseline_cache.hits - or cache_info.misses > baseline_cache.misses + with self.subTest(incremented_cache=incremented_cache): + self.assertGreater( + incremented_cache.hits + incremented_cache.misses, + baseline_cache.hits + baseline_cache.misses, ) astroid.MANAGER.clear_cache() # also calls bootstrap() # The cache sizes are now as low or lower than the original baseline - for lru_cache, baseline_cache in zip( - astroid.MANAGER._lru_caches, baseline_cache_infos + cleared_cache_infos = [lru.cache_info() for lru in astroid.MANAGER._lru_caches] + for cleared_cache, baseline_cache in zip( + cleared_cache_infos, baseline_cache_infos ): - with self.subTest(lru_cache=lru_cache): - cache_info = lru_cache.cache_info() + with self.subTest(cleared_cache=cleared_cache): # less equal because the "baseline" might have had multiple calls to bootstrap() - self.assertLessEqual(cache_info.currsize, baseline_cache.currsize) + self.assertLessEqual(cleared_cache.currsize, baseline_cache.currsize) if __name__ == "__main__": diff --git a/tests/unittest_scoped_nodes.py b/tests/unittest_scoped_nodes.py index 6d1652eb10..3cad6e0786 100644 --- a/tests/unittest_scoped_nodes.py +++ b/tests/unittest_scoped_nodes.py @@ -1678,10 +1678,8 @@ def __init__(self): "FinalClass", "ClassB", "MixinB", - "", "ClassA", "MixinA", - "", "Base", "object", ], From 5fd833afeb0be814d025e71a59650f63bf183388 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 20 Apr 2022 16:49:45 -0400 Subject: [PATCH 07/19] Ensure no behavior change with path caching --- astroid/manager.py | 4 ++-- astroid/modutils.py | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/astroid/manager.py b/astroid/manager.py index 879fa12d89..67408cf8c2 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -17,7 +17,7 @@ from astroid.interpreter._import import spec from astroid.modutils import ( NoSourceFile, - _cache_normalize_path, + _cache_normalize_path_, file_info_from_modpath, get_source_file, is_module_name_part_of_extension_package_whitelist, @@ -66,7 +66,7 @@ def __init__(self): # pylint: disable=import-outside-top-level self._lru_caches: List[_lru_cache_wrapper] = [ LookupMixIn.lookup, - _cache_normalize_path, + _cache_normalize_path_, ObjectModel.attributes, ] diff --git a/astroid/modutils.py b/astroid/modutils.py index 1c045218dc..6a51de735e 100644 --- a/astroid/modutils.py +++ b/astroid/modutils.py @@ -140,12 +140,18 @@ def _handle_blacklist(blacklist, dirnames, filenames): @lru_cache() +def _cache_normalize_path_(path: str) -> str: + return _normalize_path(path) + + def _cache_normalize_path(path: str) -> str: """Normalize path with caching.""" # _module_file calls abspath on every path in sys.path every time it's # called; on a larger codebase this easily adds up to half a second just # assembling path components. This cache alleviates that. - return _normalize_path(path) + if not path: # don't cache result for '' + return _normalize_path(path) + return _cache_normalize_path_(path) def load_module_from_name(dotted_name: str) -> types.ModuleType: From c669f8d8598baef61db16c1072e497b6bac10941 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 20 Apr 2022 16:54:53 -0400 Subject: [PATCH 08/19] try AstroidCacheSetupMixin? --- tests/unittest_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index cfbe82d1eb..2539b609e4 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -318,7 +318,7 @@ def test_borg(self) -> None: self.assertIs(built, second_built) -class ClearCacheTest(unittest.TestCase): +class ClearCacheTest(unittest.TestCase, resources.AstroidCacheSetupMixin): def test_clear_cache(self) -> None: # Get a baseline for the size of the cache after simply calling bootstrap() baseline_cache_infos = [lru.cache_info() for lru in astroid.MANAGER._lru_caches] From b5034cdba5717ff767435ab11054e44d276ca91a Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 20 Apr 2022 17:09:11 -0400 Subject: [PATCH 09/19] typo in disable --- astroid/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroid/manager.py b/astroid/manager.py index 67408cf8c2..ed3bdad0d0 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -60,7 +60,7 @@ class AstroidManager: } max_inferable_values: ClassVar[int] = 100 - def __init__(self): # pylint: disable=import-outside-top-level + def __init__(self): # pylint: disable=import-outside-toplevel from astroid.interpreter.objectmodel import ObjectModel from astroid.nodes.node_classes import LookupMixIn From e83701c1cf79b441247b8f94d3057b573ff11c3b Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 20 Apr 2022 17:19:27 -0400 Subject: [PATCH 10/19] get this disable in the right place. why only on CI? --- astroid/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astroid/manager.py b/astroid/manager.py index ed3bdad0d0..44f481ba16 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -60,7 +60,8 @@ class AstroidManager: } max_inferable_values: ClassVar[int] = 100 - def __init__(self): # pylint: disable=import-outside-toplevel + def __init__(self): + # pylint: disable=import-outside-toplevel from astroid.interpreter.objectmodel import ObjectModel from astroid.nodes.node_classes import LookupMixIn From 012f32f40a77c89743bbbe22f23528b77e9f7b95 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Thu, 21 Apr 2022 07:47:08 -0400 Subject: [PATCH 11/19] Move imports into clear_cache --- astroid/manager.py | 24 +++++++++--------------- tests/unittest_manager.py | 15 +++++++++------ tests/unittest_scoped_nodes.py | 2 ++ 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/astroid/manager.py b/astroid/manager.py index 44f481ba16..f54418bc40 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -10,7 +10,6 @@ import os import types import zipimport -from functools import _lru_cache_wrapper from typing import TYPE_CHECKING, ClassVar, List, Optional from astroid.exceptions import AstroidBuildingError, AstroidImportError @@ -61,16 +60,6 @@ class AstroidManager: max_inferable_values: ClassVar[int] = 100 def __init__(self): - # pylint: disable=import-outside-toplevel - from astroid.interpreter.objectmodel import ObjectModel - from astroid.nodes.node_classes import LookupMixIn - - self._lru_caches: List[_lru_cache_wrapper] = [ - LookupMixIn.lookup, - _cache_normalize_path_, - ObjectModel.attributes, - ] - # NOTE: cache entries are added by the [re]builder self.astroid_cache = AstroidManager.brain["astroid_cache"] self._mod_file_cache = AstroidManager.brain["_mod_file_cache"] @@ -370,15 +359,20 @@ def bootstrap(self): def clear_cache(self): """Clear the underlying caches. Also bootstraps the builtins module.""" - from astroid.inference_tip import ( # pylint: disable=import-outside-toplevel - clear_inference_tip_cache, - ) + # pylint: disable=import-outside-toplevel + from astroid.inference_tip import clear_inference_tip_cache + from astroid.interpreter.objectmodel import ObjectModel + from astroid.nodes.node_classes import LookupMixIn clear_inference_tip_cache() self.astroid_cache.clear() - for lru_cache in self._lru_caches: + for lru_cache in [ + LookupMixIn.lookup, + _cache_normalize_path_, + ObjectModel.attributes, + ]: lru_cache.cache_clear() self.bootstrap() diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 2539b609e4..13fd984f8b 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -16,7 +16,6 @@ from astroid import manager, test_utils from astroid.const import IS_JYTHON from astroid.exceptions import AstroidBuildingError, AstroidImportError -from astroid.interpreter.objectmodel import ObjectModel from astroid.modutils import is_standard_module from astroid.nodes.scoped_nodes import ClassDef @@ -320,8 +319,14 @@ def test_borg(self) -> None: class ClearCacheTest(unittest.TestCase, resources.AstroidCacheSetupMixin): def test_clear_cache(self) -> None: + from astroid.interpreter.objectmodel import ObjectModel + from astroid.modutils import _cache_normalize_path_ + from astroid.nodes.node_classes import LookupMixIn + + lrus = [LookupMixIn.lookup, _cache_normalize_path_, ObjectModel.attributes] + # Get a baseline for the size of the cache after simply calling bootstrap() - baseline_cache_infos = [lru.cache_info() for lru in astroid.MANAGER._lru_caches] + baseline_cache_infos = [lru.cache_info() for lru in lrus] # Generate some hits and misses ClassDef().lookup("garbage") @@ -329,9 +334,7 @@ def test_clear_cache(self) -> None: ObjectModel().attributes() # Did the hits or misses actually happen? - incremented_cache_infos = [ - lru.cache_info() for lru in astroid.MANAGER._lru_caches - ] + incremented_cache_infos = [lru.cache_info() for lru in lrus] for incremented_cache, baseline_cache in zip( incremented_cache_infos, baseline_cache_infos ): @@ -344,7 +347,7 @@ def test_clear_cache(self) -> None: astroid.MANAGER.clear_cache() # also calls bootstrap() # The cache sizes are now as low or lower than the original baseline - cleared_cache_infos = [lru.cache_info() for lru in astroid.MANAGER._lru_caches] + cleared_cache_infos = [lru.cache_info() for lru in lrus] for cleared_cache, baseline_cache in zip( cleared_cache_infos, baseline_cache_infos ): diff --git a/tests/unittest_scoped_nodes.py b/tests/unittest_scoped_nodes.py index 3cad6e0786..6d1652eb10 100644 --- a/tests/unittest_scoped_nodes.py +++ b/tests/unittest_scoped_nodes.py @@ -1678,8 +1678,10 @@ def __init__(self): "FinalClass", "ClassB", "MixinB", + "", "ClassA", "MixinA", + "", "Base", "object", ], From 7bbc7025afae2b424e1aa7e9627a4bdfb015896c Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Thu, 21 Apr 2022 08:12:11 -0400 Subject: [PATCH 12/19] add disable in test --- tests/unittest_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 13fd984f8b..e83e272388 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -319,11 +319,12 @@ def test_borg(self) -> None: class ClearCacheTest(unittest.TestCase, resources.AstroidCacheSetupMixin): def test_clear_cache(self) -> None: + # pylint: disable=import-outside-toplevel from astroid.interpreter.objectmodel import ObjectModel from astroid.modutils import _cache_normalize_path_ from astroid.nodes.node_classes import LookupMixIn - lrus = [LookupMixIn.lookup, _cache_normalize_path_, ObjectModel.attributes] + lrus = (LookupMixIn.lookup, _cache_normalize_path_, ObjectModel.attributes) # Get a baseline for the size of the cache after simply calling bootstrap() baseline_cache_infos = [lru.cache_info() for lru in lrus] From 8ee0632af1786b9646db83a98c87d3232d92e7b9 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Thu, 21 Apr 2022 09:14:06 -0400 Subject: [PATCH 13/19] Clarify reason for late import Co-authored-by: Pierre Sassoulas --- astroid/manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/astroid/manager.py b/astroid/manager.py index f54418bc40..b26bf68cfb 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -360,6 +360,7 @@ def bootstrap(self): def clear_cache(self): """Clear the underlying caches. Also bootstraps the builtins module.""" # pylint: disable=import-outside-toplevel + # import here because of cyclic imports from astroid.inference_tip import clear_inference_tip_cache from astroid.interpreter.objectmodel import ObjectModel from astroid.nodes.node_classes import LookupMixIn From 46d82356dde18fc5dea600530b60e7a32d876604 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Fri, 22 Apr 2022 23:05:48 -0400 Subject: [PATCH 14/19] fix disable --- astroid/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroid/manager.py b/astroid/manager.py index b26bf68cfb..5116f6545a 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -359,8 +359,8 @@ def bootstrap(self): def clear_cache(self): """Clear the underlying caches. Also bootstraps the builtins module.""" - # pylint: disable=import-outside-toplevel # import here because of cyclic imports + # pylint: disable=import-outside-toplevel from astroid.inference_tip import clear_inference_tip_cache from astroid.interpreter.objectmodel import ObjectModel from astroid.nodes.node_classes import LookupMixIn From b8ac347d0258296965b2c6607697e81cd992fa6d Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Fri, 22 Apr 2022 23:13:22 -0400 Subject: [PATCH 15/19] use tuple --- astroid/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroid/manager.py b/astroid/manager.py index 5116f6545a..30718e24a6 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -369,11 +369,11 @@ def clear_cache(self): self.astroid_cache.clear() - for lru_cache in [ + for lru_cache in ( LookupMixIn.lookup, _cache_normalize_path_, ObjectModel.attributes, - ]: + ): lru_cache.cache_clear() self.bootstrap() From a88b481e5614e8cf110ea1e51d7236b917ac4173 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Thu, 28 Apr 2022 09:47:52 -0400 Subject: [PATCH 16/19] better test method name --- tests/unittest_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index e83e272388..4ed81c0431 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -318,7 +318,7 @@ def test_borg(self) -> None: class ClearCacheTest(unittest.TestCase, resources.AstroidCacheSetupMixin): - def test_clear_cache(self) -> None: + def test_clear_cache_clears_other_lru_caches(self) -> None: # pylint: disable=import-outside-toplevel from astroid.interpreter.objectmodel import ObjectModel from astroid.modutils import _cache_normalize_path_ From 6f98e3027dc27d11249d2fc4b11179c99bba796e Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Fri, 6 May 2022 08:43:14 -0400 Subject: [PATCH 17/19] Avoid late imports --- tests/unittest_manager.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 83978cf222..96239233b3 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -320,12 +320,11 @@ def test_borg(self) -> None: class ClearCacheTest(unittest.TestCase, resources.AstroidCacheSetupMixin): def test_clear_cache_clears_other_lru_caches(self) -> None: - # pylint: disable=import-outside-toplevel - from astroid.interpreter.objectmodel import ObjectModel - from astroid.modutils import _cache_normalize_path_ - from astroid.nodes.node_classes import LookupMixIn - - lrus = (LookupMixIn.lookup, _cache_normalize_path_, ObjectModel.attributes) + lrus = ( + astroid.nodes.node_classes.LookupMixIn.lookup, + astroid.modutils._cache_normalize_path_, + astroid.interpreter.objectmodel.ObjectModel.attributes, + ) # Get a baseline for the size of the cache after simply calling bootstrap() baseline_cache_infos = [lru.cache_info() for lru in lrus] @@ -333,7 +332,7 @@ def test_clear_cache_clears_other_lru_caches(self) -> None: # Generate some hits and misses ClassDef().lookup("garbage") is_standard_module("unittest", std_path=["garbage_path"]) - ObjectModel().attributes() + astroid.interpreter.objectmodel.ObjectModel().attributes() # Did the hits or misses actually happen? incremented_cache_infos = [lru.cache_info() for lru in lrus] From e0797d4b4256809252f4b9f127e3dab1ef1267f7 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Fri, 6 May 2022 08:54:07 -0400 Subject: [PATCH 18/19] Add coverage --- tests/unittest_modutils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittest_modutils.py b/tests/unittest_modutils.py index 3fb92a845b..4cb9b6c784 100644 --- a/tests/unittest_modutils.py +++ b/tests/unittest_modutils.py @@ -288,6 +288,7 @@ def test_custom_path(self) -> None: self.assertTrue( modutils.is_standard_module("data.module", (os.path.abspath(datadir),)) ) + self.assertTrue(modutils.is_standard_module("data.module", ("",))) def test_failing_edge_cases(self) -> None: # using a subpackage/submodule path as std_path argument From ea74ae49cdadcf08520baaf1bd727a246a294fda Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Fri, 6 May 2022 09:13:57 -0400 Subject: [PATCH 19/19] Fix test isolation issue --- tests/unittest_modutils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unittest_modutils.py b/tests/unittest_modutils.py index 4cb9b6c784..3d15fb632c 100644 --- a/tests/unittest_modutils.py +++ b/tests/unittest_modutils.py @@ -179,7 +179,7 @@ def test_load_packages_without_init(self) -> None: https://github.com/PyCQA/astroid/issues/1327 """ tmp_dir = Path(tempfile.gettempdir()) - self.addCleanup(os.chdir, os.curdir) + self.addCleanup(os.chdir, os.getcwd()) os.chdir(tmp_dir) self.addCleanup(shutil.rmtree, tmp_dir / "src") @@ -288,6 +288,7 @@ def test_custom_path(self) -> None: self.assertTrue( modutils.is_standard_module("data.module", (os.path.abspath(datadir),)) ) + # "" will evaluate to cwd self.assertTrue(modutils.is_standard_module("data.module", ("",))) def test_failing_edge_cases(self) -> None: