Skip to content

Commit

Permalink
gh-91162: Support splitting of unpacked arbitrary-length tuple over T…
Browse files Browse the repository at this point in the history 
…ypeVar and TypeVarTuple parameters (alt) (GH-93412)

For example:

  A[T, *Ts][*tuple[int, ...]] -> A[int, *tuple[int, ...]]
  A[*Ts, T][*tuple[int, ...]] -> A[*tuple[int, ...], int]
  • Loading branch information
@serhiy-storchaka
serhiy-storchaka authored Jun 12, 2022
1 parent 23c9feb commit 3473817
Show file tree
Hide file tree
Showing 6 changed files with 106 additions and 127 deletions.
1 change: 1 addition & 0 deletions Include/internal/pycore_global_strings.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(__truediv__)
STRUCT_FOR_ID(__trunc__)
STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__)
STRUCT_FOR_ID(__typing_prepare_subst__)
STRUCT_FOR_ID(__typing_subst__)
STRUCT_FOR_ID(__typing_unpacked_tuple_args__)
STRUCT_FOR_ID(__warningregistry__)
Expand Down
1 change: 1 addition & 0 deletions Include/internal/pycore_runtime_init.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,7 @@ extern "C" {
INIT_ID(__truediv__), \
INIT_ID(__trunc__), \
INIT_ID(__typing_is_unpacked_typevartuple__), \
INIT_ID(__typing_prepare_subst__), \
INIT_ID(__typing_subst__), \
INIT_ID(__typing_unpacked_tuple_args__), \
INIT_ID(__warningregistry__), \
Expand Down
17 changes: 10 additions & 7 deletions Lib/test/test_typing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -753,14 +753,11 @@ class C(Generic[*Ts]): pass
('generic[*Ts]', '[*tuple_type[int]]', 'generic[int]'),
('generic[*Ts]', '[*tuple_type[*Ts]]', 'generic[*Ts]'),
('generic[*Ts]', '[*tuple_type[int, str]]', 'generic[int, str]'),
('generic[*Ts]', '[str, *tuple_type[int, ...], bool]', 'generic[str, *tuple_type[int, ...], bool]'),
('generic[*Ts]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'),
('generic[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'),
('generic[*Ts]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...]]'),

# Technically, multiple unpackings are forbidden by PEP 646, but we
# choose to be less restrictive at runtime, to allow folks room
# to experiment. So all three of these should be valid.
('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'generic[*tuple_type[int, ...], *tuple_type[str, ...]]'),
('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'),

('generic[*Ts]', '[*Ts]', 'generic[*Ts]'),
('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'),
Expand All @@ -772,15 +769,21 @@ class C(Generic[*Ts]): pass
('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'),
('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'),

('generic[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be generic[int, *tuple[int, ...]]

('generic[*Ts, T]', '[int]', 'generic[int]'),
('generic[*Ts, T]', '[int, str]', 'generic[int, str]'),
('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'),
('generic[*Ts, list[T]]', '[int]', 'generic[list[int]]'),
('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'),
('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'),

('generic[T, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...]]'),
('generic[*Ts, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], int]'),
('generic[T1, *Ts, T2]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...], int]'),
('generic[T, str, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, str, *tuple_type[int, ...]]'),
('generic[*Ts, str, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], str, int]'),
('generic[list[T], *Ts]', '[*tuple_type[int, ...]]', 'generic[list[int], *tuple_type[int, ...]]'),
('generic[*Ts, list[T]]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], list[int]]'),

('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'),
('generic[T1, T2, *tuple_type[int, ...]]', '[str, bool]', 'generic[str, bool, *tuple_type[int, ...]]'),
('generic[T1, *tuple_type[int, ...], T2]', '[str, bool]', 'generic[str, *tuple_type[int, ...], bool]'),
Expand Down
110 changes: 46 additions & 64 deletions Lib/typing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1065,6 +1065,42 @@ def __repr__(self):
def __typing_subst__(self, arg):
raise TypeError("Substitution of bare TypeVarTuple is not supported")

def __typing_prepare_subst__(self, alias, args):
params = alias.__parameters__
typevartuple_index = params.index(self)
for param in enumerate(params[typevartuple_index + 1:]):
if isinstance(param, TypeVarTuple):
raise TypeError(f"More than one TypeVarTuple parameter in {alias}")

alen = len(args)
plen = len(params)
left = typevartuple_index
right = plen - typevartuple_index - 1
var_tuple_index = None
fillarg = None
for k, arg in enumerate(args):
if not (isinstance(arg, type) and not isinstance(arg, GenericAlias)):
subargs = getattr(arg, '__typing_unpacked_tuple_args__', None)
if subargs and len(subargs) == 2 and subargs[-1] is ...:
if var_tuple_index is not None:
raise TypeError("More than one unpacked arbitrary-length tuple argument")
var_tuple_index = k
fillarg = subargs[0]
if var_tuple_index is not None:
left = min(left, var_tuple_index)
right = min(right, alen - var_tuple_index - 1)
elif left + right > alen:
raise TypeError(f"Too few arguments for {alias};"
f" actual {alen}, expected at least {plen-1}")

return (
*args[:left],
*([fillarg]*(typevartuple_index - left)),
tuple(args[left: alen - right]),
*([fillarg]*(plen - right - left - typevartuple_index - 1)),
*args[alen - right:],
)


class ParamSpecArgs(_Final, _Immutable, _root=True):
"""The args for a ParamSpec object.
Expand Down Expand Up @@ -1184,6 +1220,8 @@ def __typing_subst__(self, arg):
f"ParamSpec, or Concatenate. Got {arg}")
return arg

def __typing_prepare_subst__(self, alias, args):
return _prepare_paramspec_params(alias, args)

def _is_dunder(attr):
return attr.startswith('__') and attr.endswith('__')
Expand Down Expand Up @@ -1255,44 +1293,6 @@ def __dir__(self):
+ [attr for attr in dir(self.__origin__) if not _is_dunder(attr)]))


def _is_unpacked_tuple(x: Any) -> bool:
# Is `x` something like `*tuple[int]` or `*tuple[int, ...]`?
if not isinstance(x, _UnpackGenericAlias):
return False
# Alright, `x` is `Unpack[something]`.

# `x` will always have `__args__`, because Unpack[] and Unpack[()]
# aren't legal.
unpacked_type = x.__args__[0]

return getattr(unpacked_type, '__origin__', None) is tuple


def _is_unpacked_arbitrary_length_tuple(x: Any) -> bool:
if not _is_unpacked_tuple(x):
return False
unpacked_tuple = x.__args__[0]

if not hasattr(unpacked_tuple, '__args__'):
# It's `Unpack[tuple]`. We can't make any assumptions about the length
# of the tuple, so it's effectively an arbitrary-length tuple.
return True

tuple_args = unpacked_tuple.__args__
if not tuple_args:
# It's `Unpack[tuple[()]]`.
return False

last_arg = tuple_args[-1]
if last_arg is Ellipsis:
# It's `Unpack[tuple[something, ...]]`, which is arbitrary-length.
return True

# If the arguments didn't end with an ellipsis, then it's not an
# arbitrary-length tuple.
return False


# Special typing constructs Union, Optional, Generic, Callable and Tuple
# use three special attributes for internal bookkeeping of generic types:
# * __parameters__ is a tuple of unique free type parameters of a generic
Expand Down Expand Up @@ -1385,10 +1385,6 @@ def __getitem__(self, args):
args = (args,)
args = tuple(_type_convert(p) for p in args)
args = _unpack_args(args)
if (self._paramspec_tvars
and any(isinstance(t, ParamSpec) for t in self.__parameters__)):
args = _prepare_paramspec_params(self, args)

new_args = self._determine_new_args(args)
r = self.copy_with(new_args)
return r
Expand All @@ -1410,30 +1406,16 @@ def _determine_new_args(self, args):

params = self.__parameters__
# In the example above, this would be {T3: str}
new_arg_by_param = {}
typevartuple_index = None
for i, param in enumerate(params):
if isinstance(param, TypeVarTuple):
if typevartuple_index is not None:
raise TypeError(f"More than one TypeVarTuple parameter in {self}")
typevartuple_index = i

for param in params:
prepare = getattr(param, '__typing_prepare_subst__', None)
if prepare is not None:
args = prepare(self, args)
alen = len(args)
plen = len(params)
if typevartuple_index is not None:
i = typevartuple_index
j = alen - (plen - i - 1)
if j < i:
raise TypeError(f"Too few arguments for {self};"
f" actual {alen}, expected at least {plen-1}")
new_arg_by_param.update(zip(params[:i], args[:i]))
new_arg_by_param[params[i]] = tuple(args[i: j])
new_arg_by_param.update(zip(params[i + 1:], args[j:]))
else:
if alen != plen:
raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};"
f" actual {alen}, expected {plen}")
new_arg_by_param.update(zip(params, args))
if alen != plen:
raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};"
f" actual {alen}, expected {plen}")
new_arg_by_param = dict(zip(params, args))

new_args = []
for old_arg in self.__args__:
Expand Down
5 changes: 5 additions & 0 deletions Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Support splitting of unpacked arbitrary-length tuple over ``TypeVar`` and
``TypeVarTuple`` parameters. For example:

* ``A[T, *Ts][*tuple[int, ...]]`` -> ``A[int, *tuple[int, ...]]``
* ``A[*Ts, T][*tuple[int, ...]]`` -> ``A[*tuple[int, ...], int]``
99 changes: 43 additions & 56 deletions Objects/genericaliasobject.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ _Py_make_parameters(PyObject *args)
a non-empty tuple, return a new reference to obj. */
static PyObject *
subs_tvars(PyObject *obj, PyObject *params,
PyObject **argitems, Py_ssize_t nargs, Py_ssize_t varparam)
PyObject **argitems, Py_ssize_t nargs)
{
PyObject *subparams;
if (_PyObject_LookupAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) {
Expand All @@ -283,28 +283,28 @@ subs_tvars(PyObject *obj, PyObject *params,
Py_DECREF(subparams);
return NULL;
}
for (Py_ssize_t i = 0, j = 0; i < nsubargs; ++i) {
Py_ssize_t j = 0;
for (Py_ssize_t i = 0; i < nsubargs; ++i) {
PyObject *arg = PyTuple_GET_ITEM(subparams, i);
Py_ssize_t iparam = tuple_index(params, nparams, arg);
if (iparam == varparam) {
j = tuple_extend(&subargs, j,
argitems + iparam, nargs - nparams + 1);
if (j < 0) {
return NULL;
}
}
else {
if (iparam >= 0) {
if (iparam > varparam) {
iparam += nargs - nparams;
if (iparam >= 0) {
PyObject *param = PyTuple_GET_ITEM(params, iparam);
arg = argitems[iparam];
if (Py_TYPE(param)->tp_iter && PyTuple_Check(arg)) { // TypeVarTuple
j = tuple_extend(&subargs, j,
&PyTuple_GET_ITEM(arg, 0),
PyTuple_GET_SIZE(arg));
if (j < 0) {
return NULL;
}
arg = argitems[iparam];
continue;
}
Py_INCREF(arg);
PyTuple_SET_ITEM(subargs, j, arg);
j++;
}
Py_INCREF(arg);
PyTuple_SET_ITEM(subargs, j, arg);
j++;
}
assert(j == PyTuple_GET_SIZE(subargs));

obj = PyObject_GetItem(obj, subargs);

Expand Down Expand Up @@ -409,39 +409,37 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
self);
}
item = _unpack_args(item);
int is_tuple = PyTuple_Check(item);
Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1;
PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item;
Py_ssize_t varparam = nparams;
for (Py_ssize_t i = 0; i < nparams; i++) {
PyObject *param = PyTuple_GET_ITEM(parameters, i);
if (Py_TYPE(param)->tp_iter) { // TypeVarTuple
if (varparam < nparams) {
Py_DECREF(item);
return PyErr_Format(PyExc_TypeError,
"More than one TypeVarTuple parameter in %S",
self);
}
varparam = i;
}
}
if (varparam < nparams) {
if (nitems < nparams - 1) {
PyObject *prepare, *tmp;
if (_PyObject_LookupAttr(param, &_Py_ID(__typing_prepare_subst__), &prepare) < 0) {
Py_DECREF(item);
return PyErr_Format(PyExc_TypeError,
"Too few arguments for %R",
self);
return NULL;
}
}
else {
if (nitems != nparams) {
Py_DECREF(item);
return PyErr_Format(PyExc_TypeError,
"Too %s arguments for %R; actual %zd, expected %zd",
nitems > nparams ? "many" : "few",
self, nitems, nparams);
if (prepare && prepare != Py_None) {
if (PyTuple_Check(item)) {
tmp = PyObject_CallFunction(prepare, "OO", self, item);
}
else {
tmp = PyObject_CallFunction(prepare, "O(O)", self, item);
}
Py_DECREF(prepare);
Py_SETREF(item, tmp);
if (item == NULL) {
return NULL;
}
}
}
int is_tuple = PyTuple_Check(item);
Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1;
PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item;
if (nitems != nparams) {
Py_DECREF(item);
return PyErr_Format(PyExc_TypeError,
"Too %s arguments for %R; actual %zd, expected %zd",
nitems > nparams ? "many" : "few",
self, nitems, nparams);
}
/* Replace all type variables (specified by parameters)
with corresponding values specified by argitems.
t = list[T]; t[int] -> newargs = [int]
Expand Down Expand Up @@ -471,22 +469,11 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
if (subst) {
Py_ssize_t iparam = tuple_index(parameters, nparams, arg);
assert(iparam >= 0);
if (iparam == varparam) {
Py_DECREF(subst);
Py_DECREF(newargs);
Py_DECREF(item);
PyErr_SetString(PyExc_TypeError,
"Substitution of bare TypeVarTuple is not supported");
return NULL;
}
if (iparam > varparam) {
iparam += nitems - nparams;
}
arg = PyObject_CallOneArg(subst, argitems[iparam]);
Py_DECREF(subst);
}
else {
arg = subs_tvars(arg, parameters, argitems, nitems, varparam);
arg = subs_tvars(arg, parameters, argitems, nitems);
}
if (arg == NULL) {
Py_DECREF(newargs);
Expand Down

0 comments on commit 3473817

Please sign in to comment.