From a9d72e46663ed35d47ad675930f8583bf8e65cdd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 14 Dec 2016 17:38:28 +0100 Subject: [PATCH] Runtime implementation of TypedDict extension (#2552) This was initially proposed in python/typing#322. It works on Python 2 and 3. --- extensions/mypy_extensions.py | 84 +++++++++++++++++++++--- mypy/semanal.py | 5 +- mypy/test/testextensions.py | 119 ++++++++++++++++++++++++++++++++++ runtests.py | 2 +- 4 files changed, 199 insertions(+), 11 deletions(-) create mode 100644 mypy/test/testextensions.py diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index db66f586f051..248da3de4aae 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -7,16 +7,84 @@ # NOTE: This module must support Python 2.7 in addition to Python 3.x +import sys +# _type_check is NOT a part of public typing API, it is used here only to mimic +# the (convenient) behavior of types provided by typing module. +from typing import _type_check # type: ignore -def TypedDict(typename, fields): - """TypedDict creates a dictionary type that expects all of its + +def _check_fails(cls, other): + try: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + # Typed dicts are only for static structural subtyping. + raise TypeError('TypedDict does not support instance and class checks') + except (AttributeError, ValueError): + pass + return False + +def _dict_new(cls, *args, **kwargs): + return dict(*args, **kwargs) + +def _typeddict_new(cls, _typename, _fields=None, **kwargs): + if _fields is None: + _fields = kwargs + elif kwargs: + raise TypeError("TypedDict takes either a dict or keyword arguments," + " but not both") + return _TypedDictMeta(_typename, (), {'__annotations__': dict(_fields)}) + +class _TypedDictMeta(type): + def __new__(cls, name, bases, ns): + # Create new typed dict class object. + # This method is called directly when TypedDict is subclassed, + # or via _typeddict_new when TypedDict is instantiated. This way + # TypedDict supports all three syntaxes described in its docstring. + # Subclasses and instanes of TypedDict return actual dictionaries + # via _dict_new. + ns['__new__'] = _typeddict_new if name == 'TypedDict' else _dict_new + tp_dict = super(_TypedDictMeta, cls).__new__(cls, name, (dict,), ns) + try: + # Setting correct module is necessary to make typed dict classes pickleable. + tp_dict.__module__ = sys._getframe(2).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + anns = ns.get('__annotations__', {}) + msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" + anns = {n: _type_check(tp, msg) for n, tp in anns.items()} + for base in bases: + anns.update(base.__dict__.get('__annotations__', {})) + tp_dict.__annotations__ = anns + return tp_dict + + __instancecheck__ = __subclasscheck__ = _check_fails + + +TypedDict = _TypedDictMeta('TypedDict', (dict,), {}) +TypedDict.__module__ = __name__ +TypedDict.__doc__ = \ + """A simple typed name space. At runtime it is equivalent to a plain dict. + + TypedDict creates a dictionary type that expects all of its instances to have a certain set of keys, with each key associated with a value of a consistent type. This expectation is not checked at runtime but is only enforced by typecheckers. - """ - def new_dict(*args, **kwargs): - return dict(*args, **kwargs) + Usage:: + + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) + a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK + b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + + The type info could be accessed via Point2D.__annotations__. TypedDict + supports two additional equivalent forms:: - new_dict.__name__ = typename - new_dict.__supertype__ = dict - return new_dict + Point2D = TypedDict('Point2D', x=int, y=int, label=str) + + class Point2D(TypedDict): + x: int + y: int + label: str + + The latter syntax is only supported in Python 3.6+, while two other + syntax forms work for Python 2.7 and 3.2+ + """ diff --git a/mypy/semanal.py b/mypy/semanal.py index 77dd401ece10..5b14320e45bd 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1873,8 +1873,9 @@ def process_typeddict_definition(self, s: AssignmentStmt) -> None: return # Yes, it's a valid TypedDict definition. Add it to the symbol table. node = self.lookup(name, s) - node.kind = GDEF # TODO locally defined TypedDict - node.node = typed_dict + if node: + node.kind = GDEF # TODO locally defined TypedDict + node.node = typed_dict def check_typeddict(self, node: Expression, var_name: str = None) -> Optional[TypeInfo]: """Check if a call defines a TypedDict. diff --git a/mypy/test/testextensions.py b/mypy/test/testextensions.py new file mode 100644 index 000000000000..eca45d7e54dd --- /dev/null +++ b/mypy/test/testextensions.py @@ -0,0 +1,119 @@ +import sys +import pickle +import typing +try: + import collections.abc as collections_abc +except ImportError: + import collections as collections_abc # type: ignore # PY32 and earlier +from unittest import TestCase, main, skipUnless +sys.path[0:0] = ['extensions'] +from mypy_extensions import TypedDict + + +class BaseTestCase(TestCase): + + def assertIsSubclass(self, cls, class_or_tuple, msg=None): + if not issubclass(cls, class_or_tuple): + message = '%r is not a subclass of %r' % (cls, class_or_tuple) + if msg is not None: + message += ' : %s' % msg + raise self.failureException(message) + + def assertNotIsSubclass(self, cls, class_or_tuple, msg=None): + if issubclass(cls, class_or_tuple): + message = '%r is a subclass of %r' % (cls, class_or_tuple) + if msg is not None: + message += ' : %s' % msg + raise self.failureException(message) + + +PY36 = sys.version_info[:2] >= (3, 6) + +PY36_TESTS = """ +Label = TypedDict('Label', [('label', str)]) + +class Point2D(TypedDict): + x: int + y: int + +class LabelPoint2D(Point2D, Label): ... +""" + +if PY36: + exec(PY36_TESTS) + + +class TypedDictTests(BaseTestCase): + + def test_basics_iterable_syntax(self): + Emp = TypedDict('Emp', {'name': str, 'id': int}) + self.assertIsSubclass(Emp, dict) + self.assertIsSubclass(Emp, typing.MutableMapping) + self.assertNotIsSubclass(Emp, collections_abc.Sequence) + jim = Emp(name='Jim', id=1) + self.assertIs(type(jim), dict) + self.assertEqual(jim['name'], 'Jim') + self.assertEqual(jim['id'], 1) + self.assertEqual(Emp.__name__, 'Emp') + self.assertEqual(Emp.__module__, 'mypy.test.testextensions') + self.assertEqual(Emp.__bases__, (dict,)) + self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) + + def test_basics_keywords_syntax(self): + Emp = TypedDict('Emp', name=str, id=int) + self.assertIsSubclass(Emp, dict) + self.assertIsSubclass(Emp, typing.MutableMapping) + self.assertNotIsSubclass(Emp, collections_abc.Sequence) + jim = Emp(name='Jim', id=1) # type: ignore # mypy doesn't support keyword syntax yet + self.assertIs(type(jim), dict) + self.assertEqual(jim['name'], 'Jim') + self.assertEqual(jim['id'], 1) + self.assertEqual(Emp.__name__, 'Emp') + self.assertEqual(Emp.__module__, 'mypy.test.testextensions') + self.assertEqual(Emp.__bases__, (dict,)) + self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) + + def test_typeddict_errors(self): + Emp = TypedDict('Emp', {'name': str, 'id': int}) + self.assertEqual(TypedDict.__module__, 'mypy_extensions') + jim = Emp(name='Jim', id=1) + with self.assertRaises(TypeError): + isinstance({}, Emp) + with self.assertRaises(TypeError): + isinstance(jim, Emp) + with self.assertRaises(TypeError): + issubclass(dict, Emp) + with self.assertRaises(TypeError): + TypedDict('Hi', x=1) + with self.assertRaises(TypeError): + TypedDict('Hi', [('x', int), ('y', 1)]) + with self.assertRaises(TypeError): + TypedDict('Hi', [('x', int)], y=int) + + @skipUnless(PY36, 'Python 3.6 required') + def test_py36_class_syntax_usage(self): + self.assertEqual(LabelPoint2D.__annotations__, {'x': int, 'y': int, 'label': str}) # noqa + self.assertEqual(LabelPoint2D.__bases__, (dict,)) # noqa + self.assertNotIsSubclass(LabelPoint2D, typing.Sequence) # noqa + not_origin = Point2D(x=0, y=1) # noqa + self.assertEqual(not_origin['x'], 0) + self.assertEqual(not_origin['y'], 1) + other = LabelPoint2D(x=0, y=1, label='hi') # noqa + self.assertEqual(other['label'], 'hi') + + def test_pickle(self): + global EmpD # pickle wants to reference the class by name + EmpD = TypedDict('EmpD', name=str, id=int) + jane = EmpD({'name': 'jane', 'id': 37}) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + z = pickle.dumps(jane, proto) + jane2 = pickle.loads(z) + self.assertEqual(jane2, jane) + self.assertEqual(jane2, {'name': 'jane', 'id': 37}) + ZZ = pickle.dumps(EmpD, proto) + EmpDnew = pickle.loads(ZZ) + self.assertEqual(EmpDnew({'name': 'jane', 'id': 37}), jane) + + +if __name__ == '__main__': + main() diff --git a/runtests.py b/runtests.py index caf27eea4fed..5c67f37b7b7b 100755 --- a/runtests.py +++ b/runtests.py @@ -207,7 +207,7 @@ def add_imports(driver: Driver) -> None: PYTEST_FILES = ['mypy/test/{}.py'.format(name) for name in [ - 'testcheck', + 'testcheck', 'testextensions', ]]