From 97c5ce9e4e35fa3f636db3d7f935d4166ed2f451 Mon Sep 17 00:00:00 2001 From: Numeri Date: Tue, 7 Nov 2023 18:16:18 +0100 Subject: [PATCH 1/8] Replace setattr and getattr calls with __dict__ manipulations --- pyfakefs/mox3_stubout.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/pyfakefs/mox3_stubout.py b/pyfakefs/mox3_stubout.py index c3f3a881..a7dbe2b2 100644 --- a/pyfakefs/mox3_stubout.py +++ b/pyfakefs/mox3_stubout.py @@ -76,7 +76,7 @@ def smart_set(self, obj, attr_name, new_attr): not inspect.isclass(obj) and attr_name in obj.__dict__ ): orig_obj = obj - orig_attr = getattr(obj, attr_name) + orig_attr = obj.__dict__[attr_name] else: if not inspect.isclass(obj): @@ -91,21 +91,15 @@ def smart_set(self, obj, attr_name, new_attr): for cls in mro: try: orig_obj = cls - orig_attr = getattr(obj, attr_name) - except AttributeError: + orig_attr = obj.__dict__[attr_name] + except KeyError: continue if orig_attr is None: raise AttributeError("Attribute not found.") - # Calling getattr() on a staticmethod transforms it to a 'normal' - # function. We need to ensure that we put it back as a staticmethod. - old_attribute = obj.__dict__.get(attr_name) - if old_attribute is not None and isinstance(old_attribute, staticmethod): - orig_attr = staticmethod(orig_attr) # pytype: disable=not-callable - self.stubs.append((orig_obj, attr_name, orig_attr)) - setattr(orig_obj, attr_name, new_attr) + orig_obj.__dict__[attr_name] = new_attr def smart_unset_all(self): """Reverses all the SmartSet() calls. @@ -116,8 +110,8 @@ def smart_unset_all(self): """ self.stubs.reverse() - for args in self.stubs: - setattr(*args) + for obj, attr_name, old_attr in self.stubs: + obj.__dict__[attr_name] = old_attr self.stubs = [] @@ -143,7 +137,7 @@ def set(self, parent, child_name, new_child): old_child = classmethod(old_child.__func__) self.cache.append((parent, old_child, child_name)) - setattr(parent, child_name, new_child) + parent.__dict__[child_name] = new_child def unset_all(self): """Reverses all the Set() calls. @@ -158,5 +152,5 @@ def unset_all(self): self.cache.reverse() for parent, old_child, child_name in self.cache: - setattr(parent, child_name, old_child) + parent.__dict__[child_name] = old_child self.cache = [] From 1fe84411c2e5715a8acb6e1de8de2dfe07583955 Mon Sep 17 00:00:00 2001 From: Numeri Date: Wed, 8 Nov 2023 13:05:14 +0100 Subject: [PATCH 2/8] Update comment --- pyfakefs/mox3_stubout.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pyfakefs/mox3_stubout.py b/pyfakefs/mox3_stubout.py index a7dbe2b2..c71ce7bf 100644 --- a/pyfakefs/mox3_stubout.py +++ b/pyfakefs/mox3_stubout.py @@ -61,14 +61,9 @@ def smart_set(self, obj, attr_name, new_attr): This method supports the case where attr_name is a staticmethod or a classmethod of obj. - Notes: - - If obj is an instance, then it is its class that will actually be - stubbed. Note that the method Set() does not do that: if obj is - an instance, it (and not its class) will be stubbed. - - The stubbing is using the builtin getattr and setattr. So, the - __get__ and __set__ will be called when stubbing (TODO: A better - idea would probably be to manipulate obj.__dict__ instead of - getattr() and setattr()). + If obj is an instance, then it is its class that will actually be + stubbed. Note that the method Set() does not do that: if obj is an + instance, it (and not its class) will be stubbed. Raises AttributeError if the attribute cannot be found. """ From 54b9f3cbad39c6f363a028d78928146291921be2 Mon Sep 17 00:00:00 2001 From: Numeri Date: Mon, 13 Nov 2023 13:47:11 +0100 Subject: [PATCH 3/8] Fix failing unit test --- pyfakefs/mox3_stubout.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyfakefs/mox3_stubout.py b/pyfakefs/mox3_stubout.py index c71ce7bf..6e500de0 100644 --- a/pyfakefs/mox3_stubout.py +++ b/pyfakefs/mox3_stubout.py @@ -71,7 +71,10 @@ def smart_set(self, obj, attr_name, new_attr): not inspect.isclass(obj) and attr_name in obj.__dict__ ): orig_obj = obj - orig_attr = obj.__dict__[attr_name] + if attr_name in obj.__dict__: + orig_attr = obj.__dict__[attr_name] + else: + orig_attr = None else: if not inspect.isclass(obj): From 27a4856e05e35adc4ed500a88971787353ec7e79 Mon Sep 17 00:00:00 2001 From: Numeri Date: Tue, 14 Nov 2023 18:08:16 +0100 Subject: [PATCH 4/8] Universal methods for getting/setting through getattr/setattr, __dict__ or __slots__ --- pyfakefs/mox3_stubout.py | 112 ++++++++++++++++++++++++++++++--------- 1 file changed, 87 insertions(+), 25 deletions(-) diff --git a/pyfakefs/mox3_stubout.py b/pyfakefs/mox3_stubout.py index 6e500de0..eb304266 100644 --- a/pyfakefs/mox3_stubout.py +++ b/pyfakefs/mox3_stubout.py @@ -24,9 +24,66 @@ into pyfakefs. """ +from enum import Enum, auto +from typing import Any import inspect +class AccessMethod(Enum): + GETATTR = auto() + DICT = auto() + SLOT = auto() + + +def universal_getattr(obj: Any, attr_name: str) -> tuple[Any, AccessMethod]: + """Get an attribute's value in a universal way + + Can read any normal attributes, as well as attributes specified in + custom `__getattr__` methods, or stored in `obj.__dict__` or `obj.__slot__`. + Returns the attribute's value and the method used to access it, which can + then be used with the `set_attribute_safely` method to set it if needed. + + Warning: If the AccessMethod returned is AccessMethod.GETATTR, if the attribute + has wrappers, such as staticmethod or classmethod, these will be stripped. + """ + try: + attr_value = getattr(obj, attr_name) + access_method = AccessMethod.GETATTR + except AttributeError as e: + if hasattr(obj, '__dict__') and attr_name in obj.__dict__: + attr_value = obj.__dict__[attr_name] + access_method = AccessMethod.DICT + elif hasattr(obj, '__slot__') and attr_name in obj.__slot__: + attr_value = obj.__slot__[attr_name] + access_method = AccessMethod.SLOT + else: + raise e + + return attr_value, access_method + + +def universal_setattr(obj: Any, attr_name: str, attr_value: Any, access_method: AccessMethod): + """Set an attribute's value in a universal way + + Can set any normal attributes, as well as attributes stored in `obj.__dict__` + or `obj.__slot__`, depending on the value of `access_method`. + """ + if access_method == AccessMethod.GETATTR: + setattr(obj, attr_name, attr_value) + elif access_method == AccessMethod.DICT: + try: + obj.__dict__[attr_name] = attr_value + except KeyError: + raise AttributeError(f"Attribute {attr_name} not found in __dict__") + elif access_method == AccessMethod.SLOT: + try: + obj.__slot__[attr_name] = attr_value + except KeyError: + raise AttributeError(f"Attribute {attr_name} not found in __slot__") + else: + raise NotImplementedError(f'Unknown {access_method = } has not been implemented') + + class StubOutForTesting: """Sample Usage: @@ -67,15 +124,18 @@ def smart_set(self, obj, attr_name, new_attr): Raises AttributeError if the attribute cannot be found. """ + orig_obj = None + orig_attr = None + access_method = None + if inspect.ismodule(obj) or ( not inspect.isclass(obj) and attr_name in obj.__dict__ ): orig_obj = obj - if attr_name in obj.__dict__: - orig_attr = obj.__dict__[attr_name] - else: - orig_attr = None - + try: + orig_attr, access_method = universal_getattr(obj, attr_name) + except AttributeError: + pass else: if not inspect.isclass(obj): mro = list(inspect.getmro(obj.__class__)) @@ -84,20 +144,18 @@ def smart_set(self, obj, attr_name, new_attr): mro.reverse() - orig_attr = None - for cls in mro: try: orig_obj = cls - orig_attr = obj.__dict__[attr_name] - except KeyError: + orig_attr, access_method = universal_getattr(obj, attr_name) + except AttributeError: continue - if orig_attr is None: + if orig_obj is None or access_method is None: raise AttributeError("Attribute not found.") - self.stubs.append((orig_obj, attr_name, orig_attr)) - orig_obj.__dict__[attr_name] = new_attr + self.stubs.append((orig_obj, attr_name, orig_attr, access_method)) + universal_setattr(orig_obj, attr_name, new_attr, access_method) def smart_unset_all(self): """Reverses all the SmartSet() calls. @@ -108,8 +166,8 @@ def smart_unset_all(self): """ self.stubs.reverse() - for obj, attr_name, old_attr in self.stubs: - obj.__dict__[attr_name] = old_attr + for obj, attr_name, old_attr, access_method in self.stubs: + universal_setattr(obj, attr_name, old_attr, access_method) self.stubs = [] @@ -125,17 +183,21 @@ def set(self, parent, child_name, new_child): This method supports the case where child_name is a staticmethod or a classmethod of parent. """ - old_child = getattr(parent, child_name) + # Get the child value + old_child, access_method = universal_getattr(parent, child_name) + + # Try getting it again directly from the __dict__, to preserve any decorators + if child_name in parent.__dict__: + old_attribute = parent.__dict__.get(child_name) - old_attribute = parent.__dict__.get(child_name) - if old_attribute is not None: - if isinstance(old_attribute, staticmethod): - old_child = staticmethod(old_child) - elif isinstance(old_attribute, classmethod): - old_child = classmethod(old_child.__func__) + if old_attribute is not None: + if isinstance(old_attribute, staticmethod): + old_child = staticmethod(old_child) + elif isinstance(old_attribute, classmethod): + old_child = classmethod(old_child.__func__) - self.cache.append((parent, old_child, child_name)) - parent.__dict__[child_name] = new_child + self.cache.append((parent, old_child, child_name, access_method)) + universal_setattr(parent, child_name, new_child, access_method) def unset_all(self): """Reverses all the Set() calls. @@ -149,6 +211,6 @@ def unset_all(self): # undone) self.cache.reverse() - for parent, old_child, child_name in self.cache: - parent.__dict__[child_name] = old_child + for parent, old_child, child_name, access_method in self.cache: + universal_setattr(parent, child_name, old_child, access_method) self.cache = [] From 269d20d43bf4707beef92d54b6c1195fde3148b6 Mon Sep 17 00:00:00 2001 From: Numeri Date: Tue, 14 Nov 2023 19:03:24 +0100 Subject: [PATCH 5/8] Revert "Universal methods for getting/setting through getattr/setattr, __dict__ or __slots__" This reverts commit 0052919ddd1ed1e047507df1c0ae067b5bac7252. --- pyfakefs/mox3_stubout.py | 112 +++++++++------------------------------ 1 file changed, 25 insertions(+), 87 deletions(-) diff --git a/pyfakefs/mox3_stubout.py b/pyfakefs/mox3_stubout.py index eb304266..6e500de0 100644 --- a/pyfakefs/mox3_stubout.py +++ b/pyfakefs/mox3_stubout.py @@ -24,66 +24,9 @@ into pyfakefs. """ -from enum import Enum, auto -from typing import Any import inspect -class AccessMethod(Enum): - GETATTR = auto() - DICT = auto() - SLOT = auto() - - -def universal_getattr(obj: Any, attr_name: str) -> tuple[Any, AccessMethod]: - """Get an attribute's value in a universal way - - Can read any normal attributes, as well as attributes specified in - custom `__getattr__` methods, or stored in `obj.__dict__` or `obj.__slot__`. - Returns the attribute's value and the method used to access it, which can - then be used with the `set_attribute_safely` method to set it if needed. - - Warning: If the AccessMethod returned is AccessMethod.GETATTR, if the attribute - has wrappers, such as staticmethod or classmethod, these will be stripped. - """ - try: - attr_value = getattr(obj, attr_name) - access_method = AccessMethod.GETATTR - except AttributeError as e: - if hasattr(obj, '__dict__') and attr_name in obj.__dict__: - attr_value = obj.__dict__[attr_name] - access_method = AccessMethod.DICT - elif hasattr(obj, '__slot__') and attr_name in obj.__slot__: - attr_value = obj.__slot__[attr_name] - access_method = AccessMethod.SLOT - else: - raise e - - return attr_value, access_method - - -def universal_setattr(obj: Any, attr_name: str, attr_value: Any, access_method: AccessMethod): - """Set an attribute's value in a universal way - - Can set any normal attributes, as well as attributes stored in `obj.__dict__` - or `obj.__slot__`, depending on the value of `access_method`. - """ - if access_method == AccessMethod.GETATTR: - setattr(obj, attr_name, attr_value) - elif access_method == AccessMethod.DICT: - try: - obj.__dict__[attr_name] = attr_value - except KeyError: - raise AttributeError(f"Attribute {attr_name} not found in __dict__") - elif access_method == AccessMethod.SLOT: - try: - obj.__slot__[attr_name] = attr_value - except KeyError: - raise AttributeError(f"Attribute {attr_name} not found in __slot__") - else: - raise NotImplementedError(f'Unknown {access_method = } has not been implemented') - - class StubOutForTesting: """Sample Usage: @@ -124,18 +67,15 @@ def smart_set(self, obj, attr_name, new_attr): Raises AttributeError if the attribute cannot be found. """ - orig_obj = None - orig_attr = None - access_method = None - if inspect.ismodule(obj) or ( not inspect.isclass(obj) and attr_name in obj.__dict__ ): orig_obj = obj - try: - orig_attr, access_method = universal_getattr(obj, attr_name) - except AttributeError: - pass + if attr_name in obj.__dict__: + orig_attr = obj.__dict__[attr_name] + else: + orig_attr = None + else: if not inspect.isclass(obj): mro = list(inspect.getmro(obj.__class__)) @@ -144,18 +84,20 @@ def smart_set(self, obj, attr_name, new_attr): mro.reverse() + orig_attr = None + for cls in mro: try: orig_obj = cls - orig_attr, access_method = universal_getattr(obj, attr_name) - except AttributeError: + orig_attr = obj.__dict__[attr_name] + except KeyError: continue - if orig_obj is None or access_method is None: + if orig_attr is None: raise AttributeError("Attribute not found.") - self.stubs.append((orig_obj, attr_name, orig_attr, access_method)) - universal_setattr(orig_obj, attr_name, new_attr, access_method) + self.stubs.append((orig_obj, attr_name, orig_attr)) + orig_obj.__dict__[attr_name] = new_attr def smart_unset_all(self): """Reverses all the SmartSet() calls. @@ -166,8 +108,8 @@ def smart_unset_all(self): """ self.stubs.reverse() - for obj, attr_name, old_attr, access_method in self.stubs: - universal_setattr(obj, attr_name, old_attr, access_method) + for obj, attr_name, old_attr in self.stubs: + obj.__dict__[attr_name] = old_attr self.stubs = [] @@ -183,21 +125,17 @@ def set(self, parent, child_name, new_child): This method supports the case where child_name is a staticmethod or a classmethod of parent. """ - # Get the child value - old_child, access_method = universal_getattr(parent, child_name) - - # Try getting it again directly from the __dict__, to preserve any decorators - if child_name in parent.__dict__: - old_attribute = parent.__dict__.get(child_name) + old_child = getattr(parent, child_name) - if old_attribute is not None: - if isinstance(old_attribute, staticmethod): - old_child = staticmethod(old_child) - elif isinstance(old_attribute, classmethod): - old_child = classmethod(old_child.__func__) + old_attribute = parent.__dict__.get(child_name) + if old_attribute is not None: + if isinstance(old_attribute, staticmethod): + old_child = staticmethod(old_child) + elif isinstance(old_attribute, classmethod): + old_child = classmethod(old_child.__func__) - self.cache.append((parent, old_child, child_name, access_method)) - universal_setattr(parent, child_name, new_child, access_method) + self.cache.append((parent, old_child, child_name)) + parent.__dict__[child_name] = new_child def unset_all(self): """Reverses all the Set() calls. @@ -211,6 +149,6 @@ def unset_all(self): # undone) self.cache.reverse() - for parent, old_child, child_name, access_method in self.cache: - universal_setattr(parent, child_name, old_child, access_method) + for parent, old_child, child_name in self.cache: + parent.__dict__[child_name] = old_child self.cache = [] From c397626f6215abf6a9ef823015aeef246d9e8f3d Mon Sep 17 00:00:00 2001 From: Numeri Date: Tue, 14 Nov 2023 19:11:37 +0100 Subject: [PATCH 6/8] Add to release notes --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 6325397d..1b0c94ea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,8 @@ The released versions correspond to PyPI releases. ### Fixes * fixes the problem that filesystem patching was still active in the pytest logreport phase (see [#904](../../issues/904)) +* Restores compatability with PyTorch 2.0 and above, as well as with other classes that + have custom __setattr__ methods (see [#905](../../issues/905)). ## [Version 5.3.0](https://pypi.python.org/pypi/pyfakefs/5.3.0) (2023-10-11) Adds official support for Python 3.12. From 61483d44c356333b211f45415fc5bc127da409bf Mon Sep 17 00:00:00 2001 From: Numeri Date: Tue, 14 Nov 2023 19:16:47 +0100 Subject: [PATCH 7/8] Fix link --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1b0c94ea..28586d1d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,8 +6,8 @@ The released versions correspond to PyPI releases. ### Fixes * fixes the problem that filesystem patching was still active in the pytest logreport phase (see [#904](../../issues/904)) -* Restores compatability with PyTorch 2.0 and above, as well as with other classes that - have custom __setattr__ methods (see [#905](../../issues/905)). +* Restores compatability with PyTorch 2.0 and above, as well as with other + classes that have custom __setattr__ methods (see [#905](../../pull/905)). ## [Version 5.3.0](https://pypi.python.org/pypi/pyfakefs/5.3.0) (2023-10-11) Adds official support for Python 3.12. From 7a3566c9f8bb401a0bae5dc386f802820d0924f6 Mon Sep 17 00:00:00 2001 From: Numeri Date: Wed, 15 Nov 2023 16:47:58 +0100 Subject: [PATCH 8/8] Fix spelling --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 28586d1d..979d8c8c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ The released versions correspond to PyPI releases. ### Fixes * fixes the problem that filesystem patching was still active in the pytest logreport phase (see [#904](../../issues/904)) -* Restores compatability with PyTorch 2.0 and above, as well as with other +* Restores compatibility with PyTorch 2.0 and above, as well as with other classes that have custom __setattr__ methods (see [#905](../../pull/905)). ## [Version 5.3.0](https://pypi.python.org/pypi/pyfakefs/5.3.0) (2023-10-11)