From f38e02d3a3e1d62a6df982deb3b5a287665411d1 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 23 Apr 2022 13:20:29 -0400 Subject: [PATCH 1/7] Allow clear_cache() to load brain plugins --- ChangeLog | 2 ++ astroid/manager.py | 13 +++++++++++++ tests/unittest_manager.py | 9 +++++++++ 3 files changed, 24 insertions(+) diff --git a/ChangeLog b/ChangeLog index 0eec5468c1..457e50a299 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,8 @@ Release date: TBA Closes #1512 +* Allowed ``AstroidManager.clear_cache`` to reload necessary brain plugins. + * Rename ``ModuleSpec`` -> ``module_type`` constructor parameter to match attribute name and improve typing. Use ``type`` instead. diff --git a/astroid/manager.py b/astroid/manager.py index 4466279ce8..0a573b41ee 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -10,6 +10,7 @@ import os import types import zipimport +from importlib.util import find_spec, module_from_spec from typing import TYPE_CHECKING, ClassVar, List, Optional from astroid.exceptions import AstroidBuildingError, AstroidImportError @@ -362,5 +363,17 @@ def bootstrap(self): def clear_cache(self): """Clear the underlying cache. Also bootstraps the builtins module.""" + # import here because of cyclic imports + # pylint: disable=import-outside-toplevel + + from astroid import BRAIN_MODULES_DIRECTORY + self.astroid_cache.clear() self.bootstrap() + + # Load brain plugins: currently done in astroid.__init__.py + for module in BRAIN_MODULES_DIRECTORY.iterdir(): + if module.suffix == ".py": + module_spec = find_spec(f"astroid.brain.{module.stem}") + module_object = module_from_spec(module_spec) + module_spec.loader.exec_module(module_object) diff --git a/tests/unittest_manager.py b/tests/unittest_manager.py index 4785975692..6f0a7b16f4 100644 --- a/tests/unittest_manager.py +++ b/tests/unittest_manager.py @@ -16,6 +16,7 @@ from astroid import manager, test_utils from astroid.const import IS_JYTHON from astroid.exceptions import AstroidBuildingError, AstroidImportError +from astroid.nodes import Const from . import resources @@ -315,5 +316,13 @@ def test_borg(self) -> None: self.assertIs(built, second_built) +class ClearCacheTest(unittest.TestCase, resources.AstroidCacheSetupMixin): + def test_brain_plugins_reloaded_after_clearing_cache(self) -> None: + astroid.MANAGER.clear_cache() + format_call = astroid.extract_node("''.format()") + inferred = next(format_call.infer()) + self.assertIsInstance(inferred, Const) + + if __name__ == "__main__": unittest.main() From 3753a702f58862a3000ec07120301e0357dc20a9 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 27 Apr 2022 21:44:05 -0400 Subject: [PATCH 2/7] Also clear transforms --- astroid/manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/astroid/manager.py b/astroid/manager.py index 0a573b41ee..a7ab696e44 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -369,6 +369,7 @@ def clear_cache(self): from astroid import BRAIN_MODULES_DIRECTORY self.astroid_cache.clear() + AstroidManager.brain["_transform"] = TransformVisitor() self.bootstrap() # Load brain plugins: currently done in astroid.__init__.py From 77e9b5d05e43c7a710422adc6db2c1b07f0cb065 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 27 Apr 2022 21:48:38 -0400 Subject: [PATCH 3/7] update docstring --- astroid/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/astroid/manager.py b/astroid/manager.py index a7ab696e44..a04428d564 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -362,7 +362,8 @@ def bootstrap(self): raw_building._astroid_bootstrapping() def clear_cache(self): - """Clear the underlying cache. Also bootstraps the builtins module.""" + """Clear the underlying cache. Also bootstraps the builtins module and + re-registers transforms.""" # import here because of cyclic imports # pylint: disable=import-outside-toplevel From ad6ea217eda1d54e9dec9fd88caec95d54f77210 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 27 Apr 2022 21:56:53 -0400 Subject: [PATCH 4/7] Handle transforms in AstroidCacheSetupMixin --- tests/resources.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/resources.py b/tests/resources.py index 23f5a7a9f6..77783e3fd7 100644 --- a/tests/resources.py +++ b/tests/resources.py @@ -36,13 +36,13 @@ def tearDown(self) -> None: class AstroidCacheSetupMixin: - """Mixin for handling the astroid cache problems. + """Mixin for handling test isolation issues with the astroid cache. - When clearing the astroid cache, some tests fails due to + When clearing the astroid cache, some tests fail due to cache inconsistencies, where some objects had a different builtins object referenced. - This saves the builtins module and makes sure to add it - back to the astroid_cache after the tests finishes. + This saves the builtins module and TransformVisitor and + replaces them after the tests finish. The builtins module is special, since some of the transforms for a couple of its objects (str, bytes etc) are executed only once, so astroid_bootstrapping will be @@ -52,8 +52,11 @@ class AstroidCacheSetupMixin: @classmethod def setup_class(cls): cls._builtins = AstroidManager().astroid_cache.get("builtins") + cls._transforms = AstroidManager.brain["_transform"] @classmethod def teardown_class(cls): if cls._builtins: AstroidManager().astroid_cache["builtins"] = cls._builtins + if cls._transforms: + AstroidManager.brain["_transform"] = cls._transforms From f4f831019acc6d383def71a6a8bbf5b7b4a0b3f5 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Thu, 28 Apr 2022 09:38:35 -0400 Subject: [PATCH 5/7] Move BRAIN_MODULES_DIRECTORY to const --- astroid/__init__.py | 5 +---- astroid/const.py | 5 +++++ astroid/manager.py | 6 +----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/astroid/__init__.py b/astroid/__init__.py index 14a61c2a8d..41b8fffd1e 100644 --- a/astroid/__init__.py +++ b/astroid/__init__.py @@ -33,7 +33,6 @@ import functools import tokenize from importlib import import_module -from pathlib import Path # isort: off # We have an isort: off on '__version__' because the packaging need to access @@ -49,7 +48,7 @@ from astroid.bases import BaseInstance, BoundMethod, Instance, UnboundMethod from astroid.brain.helpers import register_module_extender from astroid.builder import extract_node, parse -from astroid.const import PY310_PLUS, Context, Del, Load, Store +from astroid.const import BRAIN_MODULES_DIRECTORY, PY310_PLUS, Context, Del, Load, Store from astroid.exceptions import ( AstroidBuildingError, AstroidBuildingException, @@ -193,8 +192,6 @@ tokenize._compile = functools.lru_cache()(tokenize._compile) # type: ignore[attr-defined] # load brain plugins -ASTROID_INSTALL_DIRECTORY = Path(__file__).parent -BRAIN_MODULES_DIRECTORY = ASTROID_INSTALL_DIRECTORY / "brain" for module in BRAIN_MODULES_DIRECTORY.iterdir(): if module.suffix == ".py": import_module(f"astroid.brain.{module.stem}") diff --git a/astroid/const.py b/astroid/const.py index 0cb2d09ecc..375c03d076 100644 --- a/astroid/const.py +++ b/astroid/const.py @@ -4,6 +4,7 @@ import enum import sys +from pathlib import Path PY36 = sys.version_info[:2] == (3, 6) PY38 = sys.version_info[:2] == (3, 8) @@ -30,3 +31,7 @@ class Context(enum.Enum): Load = Context.Load Store = Context.Store Del = Context.Del + + +ASTROID_INSTALL_DIRECTORY = Path(__file__).parent +BRAIN_MODULES_DIRECTORY = ASTROID_INSTALL_DIRECTORY / "brain" diff --git a/astroid/manager.py b/astroid/manager.py index a04428d564..5d0d16e39e 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -13,6 +13,7 @@ from importlib.util import find_spec, module_from_spec from typing import TYPE_CHECKING, ClassVar, List, Optional +from astroid.const import BRAIN_MODULES_DIRECTORY from astroid.exceptions import AstroidBuildingError, AstroidImportError from astroid.interpreter._import import spec from astroid.modutils import ( @@ -364,11 +365,6 @@ def bootstrap(self): def clear_cache(self): """Clear the underlying cache. Also bootstraps the builtins module and re-registers transforms.""" - # import here because of cyclic imports - # pylint: disable=import-outside-toplevel - - from astroid import BRAIN_MODULES_DIRECTORY - self.astroid_cache.clear() AstroidManager.brain["_transform"] = TransformVisitor() self.bootstrap() From fb6670c8a6f4e1b11c5ca234f67f85c4d8b1c641 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 4 May 2022 08:20:10 -0400 Subject: [PATCH 6/7] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com> --- astroid/manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/astroid/manager.py b/astroid/manager.py index 5d0d16e39e..9651bfada9 100644 --- a/astroid/manager.py +++ b/astroid/manager.py @@ -362,14 +362,14 @@ def bootstrap(self): raw_building._astroid_bootstrapping() - def clear_cache(self): - """Clear the underlying cache. Also bootstraps the builtins module and - re-registers transforms.""" + def clear_cache(self) -> None: + """Clear the underlying cache, bootstrap the builtins module and + re-register transforms.""" self.astroid_cache.clear() AstroidManager.brain["_transform"] = TransformVisitor() self.bootstrap() - # Load brain plugins: currently done in astroid.__init__.py + # Reload brain plugins. During initialisation this is done in astroid.__init__.py for module in BRAIN_MODULES_DIRECTORY.iterdir(): if module.suffix == ".py": module_spec = find_spec(f"astroid.brain.{module.stem}") From d90cf91422c75bccfe5ebdb4549ced796d277e78 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 4 May 2022 08:22:32 -0400 Subject: [PATCH 7/7] Remove ifs --- tests/resources.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/resources.py b/tests/resources.py index 77783e3fd7..5cf6d6b578 100644 --- a/tests/resources.py +++ b/tests/resources.py @@ -56,7 +56,5 @@ def setup_class(cls): @classmethod def teardown_class(cls): - if cls._builtins: - AstroidManager().astroid_cache["builtins"] = cls._builtins - if cls._transforms: - AstroidManager.brain["_transform"] = cls._transforms + AstroidManager().astroid_cache["builtins"] = cls._builtins + AstroidManager.brain["_transform"] = cls._transforms