Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

gh-91162: Support substitution of TypeVar with an unpacked variable-size tuple #93330

Closed
wants to merge 10 commits into from
Closed

gh-91162: Support substitution of TypeVar with an unpacked variable-size tuple #93330

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
cfb43b2
gh-91162: Support substitution of TypeVar with an unpacked variable-s…
serhiy-storchaka May 29, 2022
c638e0e
The C implementation.
serhiy-storchaka May 29, 2022
731d5fb
Remove unused code.
serhiy-storchaka May 29, 2022
03da794
Refactoring.
serhiy-storchaka May 30, 2022
b497df5
Merge branch 'main' into typing-subst-unpacked-vat-tuple
serhiy-storchaka May 31, 2022
55c4efb
Add a NEWS entry.
serhiy-storchaka Jun 1, 2022
7dcf277
Merge branch 'main' into typing-subst-unpacked-vat-tuple
serhiy-storchaka Jun 1, 2022
56f02d8
Merge branch 'main' into typing-subst-unpacked-vat-tuple
JelleZijlstra Jun 4, 2022
b091d97
Merge branch 'main' into typing-subst-unpacked-vat-tuple
serhiy-storchaka Jun 5, 2022
40ba6ca
Merge remote-tracking branch 'origin/typing-subst-unpacked-vat-tuple'…
serhiy-storchaka Jun 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions Lib/test/test_typing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -753,14 +753,11 @@ class C(Generic[*Ts]): pass
('generic[*Ts]', '[*tuple_type[int]]', 'generic[int]'),
('generic[*Ts]', '[*tuple_type[*Ts]]', 'generic[*Ts]'),
('generic[*Ts]', '[*tuple_type[int, str]]', 'generic[int, str]'),
('generic[*Ts]', '[str, *tuple_type[int, ...], bool]', 'generic[str, *tuple_type[int, ...], bool]'),
('generic[*Ts]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'),
('generic[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'),
('generic[*Ts]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...]]'),

# Technically, multiple unpackings are forbidden by PEP 646, but we
# choose to be less restrictive at runtime, to allow folks room
# to experiment. So all three of these should be valid.
('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'generic[*tuple_type[int, ...], *tuple_type[str, ...]]'),
('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'),

('generic[*Ts]', '[*Ts]', 'generic[*Ts]'),
('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'),
Expand All @@ -772,15 +769,21 @@ class C(Generic[*Ts]): pass
('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'),
('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'),

('generic[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be generic[int, *tuple[int, ...]]

('generic[*Ts, T]', '[int]', 'generic[int]'),
('generic[*Ts, T]', '[int, str]', 'generic[int, str]'),
('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'),
('generic[*Ts, list[T]]', '[int]', 'generic[list[int]]'),
('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'),
('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'),

('generic[T, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...]]'),
('generic[*Ts, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], int]'),
('generic[T1, *Ts, T2]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...], int]'),
('generic[T, str, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, str, *tuple_type[int, ...]]'),
('generic[*Ts, str, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], str, int]'),
('generic[list[T], *Ts]', '[*tuple_type[int, ...]]', 'generic[list[int], *tuple_type[int, ...]]'),
('generic[*Ts, list[T]]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], list[int]]'),

('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'),
('generic[T1, T2, *tuple_type[int, ...]]', '[str, bool]', 'generic[str, bool, *tuple_type[int, ...]]'),
('generic[T1, *tuple_type[int, ...], T2]', '[str, bool]', 'generic[str, *tuple_type[int, ...], bool]'),
Expand Down
67 changes: 23 additions & 44 deletions Lib/typing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1255,44 +1255,6 @@ def __dir__(self):
+ [attr for attr in dir(self.__origin__) if not _is_dunder(attr)]))


def _is_unpacked_tuple(x: Any) -> bool:
# Is `x` something like `*tuple[int]` or `*tuple[int, ...]`?
if not isinstance(x, _UnpackGenericAlias):
return False
# Alright, `x` is `Unpack[something]`.

# `x` will always have `__args__`, because Unpack[] and Unpack[()]
# aren't legal.
unpacked_type = x.__args__[0]

return getattr(unpacked_type, '__origin__', None) is tuple


def _is_unpacked_arbitrary_length_tuple(x: Any) -> bool:
if not _is_unpacked_tuple(x):
return False
unpacked_tuple = x.__args__[0]

if not hasattr(unpacked_tuple, '__args__'):
# It's `Unpack[tuple]`. We can't make any assumptions about the length
# of the tuple, so it's effectively an arbitrary-length tuple.
return True

tuple_args = unpacked_tuple.__args__
if not tuple_args:
# It's `Unpack[tuple[()]]`.
return False

last_arg = tuple_args[-1]
if last_arg is Ellipsis:
# It's `Unpack[tuple[something, ...]]`, which is arbitrary-length.
return True

# If the arguments didn't end with an ellipsis, then it's not an
# arbitrary-length tuple.
return False


# Special typing constructs Union, Optional, Generic, Callable and Tuple
# use three special attributes for internal bookkeeping of generic types:
# * __parameters__ is a tuple of unique free type parameters of a generic
Expand Down Expand Up @@ -1421,14 +1383,31 @@ def _determine_new_args(self, args):
alen = len(args)
plen = len(params)
if typevartuple_index is not None:
i = typevartuple_index
j = alen - (plen - i - 1)
if j < i:
left = typevartuple_index
right = plen - typevartuple_index - 1
var_tuple_index = None
for k, arg in enumerate(args):
if not (isinstance(arg, type) and not isinstance(arg, GenericAlias)):
subargs = getattr(arg, '__typing_unpacked_tuple_args__', None)
if subargs and len(subargs) == 2 and subargs[-1] is ...:
if var_tuple_index is not None:
raise TypeError("More than one unpacked arbitrary-length tuple argument")
var_tuple_index = k
fillarg = subargs[0]
if var_tuple_index is not None:
left = min(left, var_tuple_index)
right = min(right, alen - var_tuple_index - 1)
elif left + right > alen:
raise TypeError(f"Too few arguments for {self};"
f" actual {alen}, expected at least {plen-1}")
new_arg_by_param.update(zip(params[:i], args[:i]))
new_arg_by_param[params[i]] = tuple(args[i: j])
new_arg_by_param.update(zip(params[i + 1:], args[j:]))

new_arg_by_param.update(zip(params[:left], args[:left]))
for k in range(left, typevartuple_index):
new_arg_by_param[params[k]] = fillarg
new_arg_by_param[params[typevartuple_index]] = tuple(args[left: alen - right])
for k in range(typevartuple_index + 1, plen - right):
new_arg_by_param[params[k]] = fillarg
new_arg_by_param.update(zip(params[plen - right:], args[alen - right:]))
else:
if alen != plen:
raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};"
Expand Down
5 changes: 5 additions & 0 deletions Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Support splitting of unpacked arbitrary-length tuple over ``TypeVar`` and
``TypeVarTuple`` parameters. For example:

* A[T, *Ts][*tuple[int, ...]] -> A[int, *tuple[int, ...]]
* A[*Ts, T][*tuple[int, ...]] -> A[*tuple[int, ...], int]
94 changes: 85 additions & 9 deletions Objects/genericaliasobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,9 @@ _Py_make_parameters(PyObject *args)
a non-empty tuple, return a new reference to obj. */
static PyObject *
subs_tvars(PyObject *obj, PyObject *params,
PyObject **argitems, Py_ssize_t nargs, Py_ssize_t varparam)
PyObject **argitems, Py_ssize_t nargs,
Py_ssize_t varparam, Py_ssize_t left, Py_ssize_t right,
PyObject *fillarg)
{
PyObject *subparams;
if (_PyObject_LookupAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) {
Expand All @@ -283,28 +285,36 @@ subs_tvars(PyObject *obj, PyObject *params,
Py_DECREF(subparams);
return NULL;
}
for (Py_ssize_t i = 0, j = 0; i < nsubargs; ++i) {
Py_ssize_t j = 0;
for (Py_ssize_t i = 0; i < nsubargs; ++i) {
PyObject *arg = PyTuple_GET_ITEM(subparams, i);
Py_ssize_t iparam = tuple_index(params, nparams, arg);
if (iparam == varparam) {
j = tuple_extend(&subargs, j,
argitems + iparam, nargs - nparams + 1);
argitems + left, nargs - left - right);
if (j < 0) {
return NULL;
}
}
else {
if (iparam >= 0) {
if (iparam > varparam) {
if (iparam < left) {
arg = argitems[iparam];
}
else if (iparam >= nparams - right) {
iparam += nargs - nparams;
arg = argitems[iparam];
}
else {
arg = fillarg;
}
arg = argitems[iparam];
}
Py_INCREF(arg);
PyTuple_SET_ITEM(subargs, j, arg);
j++;
}
}
assert(j == PyTuple_GET_SIZE(subargs));

obj = PyObject_GetItem(obj, subargs);

Expand Down Expand Up @@ -399,6 +409,27 @@ _unpack_args(PyObject *item)
return newargs;
}

static PyObject *
_get_unpacked_var_tuple_arg(PyObject *arg)
{
if (PyType_Check(arg)) {
return NULL;
}
PyObject *subargs = _unpacked_tuple_args(arg);
if (subargs != NULL &&
PyTuple_Check(subargs) &&
PyTuple_GET_SIZE(subargs) == 2 &&
PyTuple_GET_ITEM(subargs, 1) == Py_Ellipsis)
{
PyObject *subarg = PyTuple_GET_ITEM(subargs, 0);
Py_INCREF(subarg);
Py_DECREF(subargs);
return subarg;
}
Py_XDECREF(subargs);
return NULL;
}

PyObject *
_Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObject *item)
{
Expand All @@ -425,8 +456,37 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
varparam = i;
}
}
PyObject *fillarg = NULL;
Py_ssize_t vartuplearg = nitems;
Py_ssize_t left = varparam;
Py_ssize_t right = nparams - varparam - 1;
if (varparam < nparams) {
if (nitems < nparams - 1) {
for (Py_ssize_t i = 0; i < nitems; i++) {
PyObject *arg = _get_unpacked_var_tuple_arg(argitems[i]);
if (arg) {
if (vartuplearg < nitems) {
Py_DECREF(arg);
Py_DECREF(fillarg);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This could use an assert(fillarg) to make sure it's not NULL.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Py_DECREF() makes sure that the argument is not NULL. The result is the same -- a crash.

Py_DECREF(item);
return PyErr_Format(PyExc_TypeError,
"More than one unpacked arbitrary-length tuple argument",
self);
}
vartuplearg = i;
fillarg = arg;
}
else if (PyErr_Occurred()) {
Py_XDECREF(fillarg);
Py_DECREF(item);
return NULL;
}
}
if (vartuplearg < nitems) {
assert(fillarg);
left = Py_MIN(left, vartuplearg);
right = Py_MIN(right, nitems - vartuplearg - 1);
}
else if (left + right > nitems) {
Py_DECREF(item);
return PyErr_Format(PyExc_TypeError,
"Too few arguments for %R",
Expand All @@ -451,6 +511,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
Py_ssize_t nargs = PyTuple_GET_SIZE(args);
PyObject *newargs = PyTuple_New(nargs);
if (newargs == NULL) {
Py_XDECREF(fillarg);
Py_DECREF(item);
return NULL;
}
Expand All @@ -459,12 +520,14 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
int unpack = _is_unpacked_typevartuple(arg);
if (unpack < 0) {
Py_DECREF(newargs);
Py_XDECREF(fillarg);
Py_DECREF(item);
return NULL;
}
PyObject *subst;
if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) {
Py_DECREF(newargs);
Py_XDECREF(fillarg);
Py_DECREF(item);
return NULL;
}
Expand All @@ -474,22 +537,33 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
if (iparam == varparam) {
Py_DECREF(subst);
Py_DECREF(newargs);
Py_XDECREF(fillarg);
Py_DECREF(item);
PyErr_SetString(PyExc_TypeError,
"Substitution of bare TypeVarTuple is not supported");
return NULL;
}
if (iparam > varparam) {
if (iparam < left) {
arg = argitems[iparam];
}
else if (iparam >= nparams - right) {
iparam += nitems - nparams;
arg = argitems[iparam];
}
else {
assert(fillarg);
arg = fillarg;
}
arg = PyObject_CallOneArg(subst, argitems[iparam]);
arg = PyObject_CallOneArg(subst, arg);
Py_DECREF(subst);
}
else {
arg = subs_tvars(arg, parameters, argitems, nitems, varparam);
arg = subs_tvars(arg, parameters, argitems, nitems,
varparam, left, right, fillarg);
}
if (arg == NULL) {
Py_DECREF(newargs);
Py_XDECREF(fillarg);
Py_DECREF(item);
return NULL;
}
Expand All @@ -498,6 +572,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
&PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg));
Py_DECREF(arg);
if (jarg < 0) {
Py_XDECREF(fillarg);
Py_DECREF(item);
return NULL;
}
Expand All @@ -508,6 +583,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
}
}

Py_XDECREF(fillarg);
Py_DECREF(item);
return newargs;
}
Expand Down