From 0dd91b48ee4f89074c95238c7477775fd6105893 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 29 Apr 2020 10:14:58 +0200 Subject: [PATCH] Fix cloudpickle incompatibilities on early Python 3.5 versions (#361) --- CHANGES.md | 4 ++ cloudpickle/cloudpickle.py | 80 +++++++++++++++++++++++++++++++------- tests/cloudpickle_test.py | 79 +++++++++++++++++++++++++++---------- 3 files changed, 130 insertions(+), 33 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ad584c7ea..17fe243a7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,10 @@ 1.4.1 (in development) ====================== +- Fix incompatibilities between cloudpickle 1.4.0 and Python 3.5.0/1/2 + introduced by the new support of cloudpickle for pickling typing constructs. + ([issue #360](https://github.com/cloudpipe/cloudpickle/issues/360)) + - Restore compat with loading dynamic classes pickled with cloudpickle version 1.2.1 that would reference the `types.ClassType` attribute. ([PR #359](https://github.com/cloudpipe/cloudpickle/pull/359)) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index fb5beb5f5..c639daab1 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -61,7 +61,7 @@ import typing from enum import Enum -from typing import Generic, Union, Tuple, Callable, ClassVar +from typing import Generic, Union, Tuple, Callable from pickle import _Pickler as Pickler from pickle import _getattribute from io import BytesIO @@ -73,6 +73,11 @@ except ImportError: _typing_extensions = Literal = Final = None +if sys.version_info >= (3, 5, 3): + from typing import ClassVar +else: # pragma: no cover + ClassVar = None + # cloudpickle is meant for inter process communication: we expect all # communicating processes to run the same Python version hence we favor @@ -432,10 +437,24 @@ def _extract_class_dict(cls): if sys.version_info[:2] < (3, 7): # pragma: no branch def _is_parametrized_type_hint(obj): # This is very cheap but might generate false positives. - origin = getattr(obj, '__origin__', None) # typing Constructs - values = getattr(obj, '__values__', None) # typing_extensions.Literal - type_ = getattr(obj, '__type__', None) # typing_extensions.Final - return origin is not None or values is not None or type_ is not None + # general typing Constructs + is_typing = getattr(obj, '__origin__', None) is not None + + # typing_extensions.Literal + is_litteral = getattr(obj, '__values__', None) is not None + + # typing_extensions.Final + is_final = getattr(obj, '__type__', None) is not None + + # typing.Union/Tuple for old Python 3.5 + is_union = getattr(obj, '__union_params__', None) is not None + is_tuple = getattr(obj, '__tuple_params__', None) is not None + is_callable = ( + getattr(obj, '__result__', None) is not None and + getattr(obj, '__args__', None) is not None + ) + return any((is_typing, is_litteral, is_final, is_union, is_tuple, + is_callable)) def _create_parametrized_type_hint(origin, args): return origin[args] @@ -971,14 +990,40 @@ def _save_parametrized_type_hint(self, obj): initargs = (Final, obj.__type__) elif type(obj) is type(ClassVar): initargs = (ClassVar, obj.__type__) - elif type(obj) in [type(Union), type(Tuple), type(Generic)]: - initargs = (obj.__origin__, obj.__args__) + elif type(obj) is type(Generic): + parameters = obj.__parameters__ + if len(obj.__parameters__) > 0: + # in early Python 3.5, __parameters__ was sometimes + # preferred to __args__ + initargs = (obj.__origin__, parameters) + else: + initargs = (obj.__origin__, obj.__args__) + elif type(obj) is type(Union): + if sys.version_info < (3, 5, 3): # pragma: no cover + initargs = (Union, obj.__union_params__) + else: + initargs = (Union, obj.__args__) + elif type(obj) is type(Tuple): + if sys.version_info < (3, 5, 3): # pragma: no cover + initargs = (Tuple, obj.__tuple_params__) + else: + initargs = (Tuple, obj.__args__) elif type(obj) is type(Callable): - args = obj.__args__ - if args[0] is Ellipsis: - initargs = (obj.__origin__, args) + if sys.version_info < (3, 5, 3): # pragma: no cover + args = obj.__args__ + result = obj.__result__ + if args != Ellipsis: + if isinstance(args, tuple): + args = list(args) + else: + args = [args] else: - initargs = (obj.__origin__, (list(args[:-1]), args[-1])) + (*args, result) = obj.__args__ + if len(args) == 1 and args[0] is Ellipsis: + args = Ellipsis + else: + args = list(args) + initargs = (Callable, (args, result)) else: # pragma: no cover raise pickle.PicklingError( "Cloudpickle Error: Unknown type {}".format(type(obj)) @@ -1301,14 +1346,23 @@ def _make_typevar(name, bound, constraints, covariant, contravariant, name, *constraints, bound=bound, covariant=covariant, contravariant=contravariant ) - return _lookup_class_or_track(class_tracker_id, tv) + if class_tracker_id is not None: + return _lookup_class_or_track(class_tracker_id, tv) + else: # pragma: nocover + # Only for Python 3.5.3 compat. + return tv def _decompose_typevar(obj): + try: + class_tracker_id = _get_or_create_tracker_id(obj) + except TypeError: # pragma: nocover + # TypeVar instances are not weakref-able in Python 3.5.3 + class_tracker_id = None return ( obj.__name__, obj.__bound__, obj.__constraints__, obj.__covariant__, obj.__contravariant__, - _get_or_create_tracker_id(obj), + class_tracker_id, ) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 126bb310a..890d4165d 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2047,6 +2047,9 @@ def test_pickle_dynamic_typevar(self): for attr in attr_list: assert getattr(T, attr) == getattr(depickled_T, attr) + @pytest.mark.skipif( + sys.version_info[:3] == (3, 5, 3), + reason="TypeVar instances are not weakref-able in Python 3.5.3") def test_pickle_dynamic_typevar_tracking(self): T = typing.TypeVar("T") T2 = subprocess_pickle_echo(T, protocol=self.protocol) @@ -2081,20 +2084,32 @@ class C(typing.Generic[T]): with subprocess_worker(protocol=self.protocol) as worker: - def check_generic(generic, origin, type_value): + def check_generic(generic, origin, type_value, use_args): assert generic.__origin__ is origin - assert len(generic.__args__) == 1 - assert generic.__args__[0] is type_value - assert len(origin.__orig_bases__) == 1 - ob = origin.__orig_bases__[0] - assert ob.__origin__ is typing.Generic + if sys.version_info >= (3, 5, 3): + assert len(origin.__orig_bases__) == 1 + ob = origin.__orig_bases__[0] + assert ob.__origin__ is typing.Generic + else: # Python 3.5.[0-1-2], pragma: no cover + assert len(origin.__bases__) == 1 + ob = origin.__bases__[0] + + if use_args: + assert len(generic.__args__) == 1 + assert generic.__args__[0] is type_value + else: + assert len(generic.__parameters__) == 1 + assert generic.__parameters__[0] is type_value assert len(ob.__parameters__) == 1 return "ok" - assert check_generic(C[int], C, int) == "ok" - assert worker.run(check_generic, C[int], C, int) == "ok" + # backward-compat for old Python 3.5 versions that sometimes relies + # on __parameters__ + use_args = getattr(C[int], '__args__', ()) != () + assert check_generic(C[int], C, int, use_args) == "ok" + assert worker.run(check_generic, C[int], C, int, use_args) == "ok" def test_locally_defined_class_with_type_hints(self): with subprocess_worker(protocol=self.protocol) as worker: @@ -2116,19 +2131,38 @@ def check_annotations(obj, expected_type): assert check_annotations(obj, type_) == "ok" assert worker.run(check_annotations, obj, type_) == "ok" - def test_generic_extensions(self): + def test_generic_extensions_literal(self): typing_extensions = pytest.importorskip('typing_extensions') - objs = [ - typing_extensions.Literal, - typing_extensions.Final, - typing_extensions.Literal['a'], - typing_extensions.Final[int], + def check_literal_equal(obj1, obj2): + assert obj1.__values__ == obj2.__values__ + assert type(obj1) == type(obj2) == typing_extensions._LiteralMeta + literal_objs = [ + typing_extensions.Literal, typing_extensions.Literal['a'] ] + for obj in literal_objs: + depickled_obj = pickle_depickle(obj, protocol=self.protocol) + if sys.version_info[:3] >= (3, 5, 3): + assert depickled_obj == obj + else: + # __eq__ does not work for Literal objects in early Python 3.5 + check_literal_equal(obj, depickled_obj) + + def test_generic_extensions_final(self): + typing_extensions = pytest.importorskip('typing_extensions') + + def check_final_equal(obj1, obj2): + assert obj1.__type__ == obj2.__type__ + assert type(obj1) == type(obj2) == typing_extensions._FinalMeta + final_objs = [typing_extensions.Final, typing_extensions.Final[int]] - for obj in objs: + for obj in final_objs: depickled_obj = pickle_depickle(obj, protocol=self.protocol) - assert depickled_obj == obj + if sys.version_info[:3] >= (3, 5, 3): + assert depickled_obj == obj + else: + # __eq__ does not work for Final objects in early Python 3.5 + check_final_equal(obj, depickled_obj) def test_class_annotations(self): class C: @@ -2182,10 +2216,10 @@ def _all_types_to_test(): class C(typing.Generic[T]): pass - return [ + types_to_test = [ C, C[int], - T, typing.Any, typing.NoReturn, typing.Optional, - typing.Generic, typing.Union, typing.ClassVar, + T, typing.Any, typing.Optional, + typing.Generic, typing.Union, typing.Optional[int], typing.Generic[T], typing.Callable[[int], typing.Any], @@ -2193,10 +2227,15 @@ class C(typing.Generic[T]): typing.Callable[[], typing.Any], typing.Tuple[int, ...], typing.Tuple[int, C[int]], - typing.ClassVar[C[int]], typing.List[int], typing.Dict[int, str], ] + if sys.version_info[:3] >= (3, 5, 3): + types_to_test.append(typing.ClassVar) + types_to_test.append(typing.ClassVar[C[int]]) + if sys.version_info >= (3, 5, 4): + types_to_test.append(typing.NoReturn) + return types_to_test if __name__ == '__main__':