Fix cloudpickle incompatibilities on early Python 3.5 versions (#361)

CHANGES.md

@@ -1,6 +1,10 @@
 1.4.1 (in development)
 ======================
 
+- Fix incompatibilities between cloudpickle 1.4.0 and Python 3.5.0/1/2
+  introduced by the new support of cloudpickle for pickling typing constructs.
+  ([issue #360](https://github.com/cloudpipe/cloudpickle/issues/360))
+
 - Restore compat with loading dynamic classes pickled with cloudpickle
   version 1.2.1 that would reference the `types.ClassType` attribute.
   ([PR #359](https://github.com/cloudpipe/cloudpickle/pull/359))

cloudpickle/cloudpickle.py

@@ -61,7 +61,7 @@
 import typing
 from enum import Enum
 
-from typing import Generic, Union, Tuple, Callable, ClassVar
+from typing import Generic, Union, Tuple, Callable
 from pickle import _Pickler as Pickler
 from pickle import _getattribute
 from io import BytesIO
@@ -73,6 +73,11 @@
 except ImportError:
     _typing_extensions = Literal = Final = None
 
+if sys.version_info >= (3, 5, 3):
+    from typing import ClassVar
+else:  # pragma: no cover
+    ClassVar = None
+
 
 # cloudpickle is meant for inter process communication: we expect all
 # communicating processes to run the same Python version hence we favor
@@ -432,10 +437,24 @@ def _extract_class_dict(cls):
 if sys.version_info[:2] < (3, 7):  # pragma: no branch
     def _is_parametrized_type_hint(obj):
         # This is very cheap but might generate false positives.
-        origin = getattr(obj, '__origin__', None)  # typing Constructs
-        values = getattr(obj, '__values__', None)  # typing_extensions.Literal
-        type_ = getattr(obj, '__type__', None)     # typing_extensions.Final
-        return origin is not None or values is not None or type_ is not None
+        # general typing Constructs
+        is_typing = getattr(obj, '__origin__', None) is not None
+
+        # typing_extensions.Literal
+        is_litteral = getattr(obj, '__values__', None) is not None
+
+        # typing_extensions.Final
+        is_final = getattr(obj, '__type__', None) is not None
+
+        # typing.Union/Tuple for old Python 3.5
+        is_union = getattr(obj, '__union_params__', None) is not None
+        is_tuple = getattr(obj, '__tuple_params__', None) is not None
+        is_callable = (
+            getattr(obj, '__result__', None) is not None and
+            getattr(obj, '__args__', None) is not None
+        )
+        return any((is_typing, is_litteral, is_final, is_union, is_tuple,
+                    is_callable))
 
     def _create_parametrized_type_hint(origin, args):
         return origin[args]
@@ -971,14 +990,40 @@ def _save_parametrized_type_hint(self, obj):
                 initargs = (Final, obj.__type__)
             elif type(obj) is type(ClassVar):
                 initargs = (ClassVar, obj.__type__)
-            elif type(obj) in [type(Union), type(Tuple), type(Generic)]:
-                initargs = (obj.__origin__, obj.__args__)
+            elif type(obj) is type(Generic):
+                parameters = obj.__parameters__
+                if len(obj.__parameters__) > 0:
+                    # in early Python 3.5, __parameters__ was sometimes
+                    # preferred to __args__
+                    initargs = (obj.__origin__, parameters)
+                else:
+                    initargs = (obj.__origin__, obj.__args__)
+            elif type(obj) is type(Union):
+                if sys.version_info < (3, 5, 3):  # pragma: no cover
+                    initargs = (Union, obj.__union_params__)
+                else:
+                    initargs = (Union, obj.__args__)
+            elif type(obj) is type(Tuple):
+                if sys.version_info < (3, 5, 3):  # pragma: no cover
+                    initargs = (Tuple, obj.__tuple_params__)
+                else:
+                    initargs = (Tuple, obj.__args__)
             elif type(obj) is type(Callable):
-                args = obj.__args__
-                if args[0] is Ellipsis:
-                    initargs = (obj.__origin__, args)
+                if sys.version_info < (3, 5, 3):  # pragma: no cover
+                    args = obj.__args__
+                    result = obj.__result__
+                    if args != Ellipsis:
+                        if isinstance(args, tuple):
+                            args = list(args)
+                        else:
+                            args = [args]
                 else:
-                    initargs = (obj.__origin__, (list(args[:-1]), args[-1]))
+                    (*args, result) = obj.__args__
+                    if len(args) == 1 and args[0] is Ellipsis:
+                        args = Ellipsis
+                    else:
+                        args = list(args)
+                initargs = (Callable, (args, result))
             else:  # pragma: no cover
                 raise pickle.PicklingError(
                     "Cloudpickle Error: Unknown type {}".format(type(obj))
@@ -1301,14 +1346,23 @@ def _make_typevar(name, bound, constraints, covariant, contravariant,
         name, *constraints, bound=bound,
         covariant=covariant, contravariant=contravariant
     )
-    return _lookup_class_or_track(class_tracker_id, tv)
+    if class_tracker_id is not None:
+        return _lookup_class_or_track(class_tracker_id, tv)
+    else:  # pragma: nocover
+        # Only for Python 3.5.3 compat.
+        return tv
 
 
 def _decompose_typevar(obj):
+    try:
+        class_tracker_id = _get_or_create_tracker_id(obj)
+    except TypeError:  # pragma: nocover
+        # TypeVar instances are not weakref-able in Python 3.5.3
+        class_tracker_id = None
     return (
         obj.__name__, obj.__bound__, obj.__constraints__,
         obj.__covariant__, obj.__contravariant__,
-        _get_or_create_tracker_id(obj),
+        class_tracker_id,
     )
 
 

tests/cloudpickle_test.py

@@ -2047,6 +2047,9 @@ def test_pickle_dynamic_typevar(self):
         for attr in attr_list:
             assert getattr(T, attr) == getattr(depickled_T, attr)
 
+    @pytest.mark.skipif(
+        sys.version_info[:3] == (3, 5, 3),
+        reason="TypeVar instances are not weakref-able in Python 3.5.3")
     def test_pickle_dynamic_typevar_tracking(self):
         T = typing.TypeVar("T")
         T2 = subprocess_pickle_echo(T, protocol=self.protocol)
@@ -2081,20 +2084,32 @@ class C(typing.Generic[T]):
 
         with subprocess_worker(protocol=self.protocol) as worker:
 
-            def check_generic(generic, origin, type_value):
+            def check_generic(generic, origin, type_value, use_args):
                 assert generic.__origin__ is origin
-                assert len(generic.__args__) == 1
-                assert generic.__args__[0] is type_value
 
-                assert len(origin.__orig_bases__) == 1
-                ob = origin.__orig_bases__[0]
-                assert ob.__origin__ is typing.Generic
+                if sys.version_info >= (3, 5, 3):
+                    assert len(origin.__orig_bases__) == 1
+                    ob = origin.__orig_bases__[0]
+                    assert ob.__origin__ is typing.Generic
+                else:  # Python 3.5.[0-1-2], pragma: no cover
+                    assert len(origin.__bases__) == 1
+                    ob = origin.__bases__[0]
+
+                if use_args:
+                    assert len(generic.__args__) == 1
+                    assert generic.__args__[0] is type_value
+                else:
+                    assert len(generic.__parameters__) == 1
+                    assert generic.__parameters__[0] is type_value
                 assert len(ob.__parameters__) == 1
 
                 return "ok"
 
-            assert check_generic(C[int], C, int) == "ok"
-            assert worker.run(check_generic, C[int], C, int) == "ok"
+            # backward-compat for old Python 3.5 versions that sometimes relies
+            # on __parameters__
+            use_args = getattr(C[int], '__args__', ()) != ()
+            assert check_generic(C[int], C, int, use_args) == "ok"
+            assert worker.run(check_generic, C[int], C, int, use_args) == "ok"
 
     def test_locally_defined_class_with_type_hints(self):
         with subprocess_worker(protocol=self.protocol) as worker:
@@ -2116,19 +2131,38 @@ def check_annotations(obj, expected_type):
                 assert check_annotations(obj, type_) == "ok"
                 assert worker.run(check_annotations, obj, type_) == "ok"
 
-    def test_generic_extensions(self):
+    def test_generic_extensions_literal(self):
         typing_extensions = pytest.importorskip('typing_extensions')
 
-        objs = [
-            typing_extensions.Literal,
-            typing_extensions.Final,
-            typing_extensions.Literal['a'],
-            typing_extensions.Final[int],
+        def check_literal_equal(obj1, obj2):
+            assert obj1.__values__ == obj2.__values__
+            assert type(obj1) == type(obj2) == typing_extensions._LiteralMeta
+        literal_objs = [
+            typing_extensions.Literal, typing_extensions.Literal['a']
         ]
+        for obj in literal_objs:
+            depickled_obj = pickle_depickle(obj, protocol=self.protocol)
+            if sys.version_info[:3] >= (3, 5, 3):
+                assert depickled_obj == obj
+            else:
+                # __eq__ does not work for Literal objects in early Python 3.5
+                check_literal_equal(obj, depickled_obj)
+
+    def test_generic_extensions_final(self):
+        typing_extensions = pytest.importorskip('typing_extensions')
+
+        def check_final_equal(obj1, obj2):
+            assert obj1.__type__ == obj2.__type__
+            assert type(obj1) == type(obj2) == typing_extensions._FinalMeta
+        final_objs = [typing_extensions.Final, typing_extensions.Final[int]]
 
-        for obj in objs:
+        for obj in final_objs:
             depickled_obj = pickle_depickle(obj, protocol=self.protocol)
-            assert depickled_obj == obj
+            if sys.version_info[:3] >= (3, 5, 3):
+                assert depickled_obj == obj
+            else:
+                # __eq__ does not work for Final objects in early Python 3.5
+                check_final_equal(obj, depickled_obj)
 
     def test_class_annotations(self):
         class C:
@@ -2182,21 +2216,26 @@ def _all_types_to_test():
     class C(typing.Generic[T]):
         pass
 
-    return [
+    types_to_test = [
         C, C[int],
-        T, typing.Any, typing.NoReturn, typing.Optional,
-        typing.Generic, typing.Union, typing.ClassVar,
+        T, typing.Any, typing.Optional,
+        typing.Generic, typing.Union,
         typing.Optional[int],
         typing.Generic[T],
         typing.Callable[[int], typing.Any],
         typing.Callable[..., typing.Any],
         typing.Callable[[], typing.Any],
         typing.Tuple[int, ...],
         typing.Tuple[int, C[int]],
-        typing.ClassVar[C[int]],
         typing.List[int],
         typing.Dict[int, str],
     ]
+    if sys.version_info[:3] >= (3, 5, 3):
+        types_to_test.append(typing.ClassVar)
+        types_to_test.append(typing.ClassVar[C[int]])
+    if sys.version_info >= (3, 5, 4):
+        types_to_test.append(typing.NoReturn)
+    return types_to_test
 
 
 if __name__ == '__main__':