From a03c0d1fe352339dd1abd1665940329fca46998e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 4 Feb 2022 09:46:17 +0200 Subject: [PATCH] bpo-44796: Unify TypeVar and ParamSpec substitution Add methods __typing_subst__() in TypeVar and ParamSpec. --- Lib/_collections_abc.py | 63 +++++------------------------- Lib/test/test_typing.py | 75 +++++++++++++++++++++++++++++------- Lib/typing.py | 72 ++++++++++++++++++---------------- Objects/genericaliasobject.c | 48 ++++++++--------------- 4 files changed, 125 insertions(+), 133 deletions(-) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 97913c77721da9..7fef4c5c990e44 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -430,25 +430,13 @@ def __new__(cls, origin, args): raise TypeError( "Callable must be used as Callable[[arg, ...], result].") t_args, t_result = args - if isinstance(t_args, list): + if isinstance(t_args, (tuple, list)): args = (*t_args, t_result) elif not _is_param_expr(t_args): raise TypeError(f"Expected a list of types, an ellipsis, " f"ParamSpec, or Concatenate. Got {t_args}") return super().__new__(cls, origin, args) - @property - def __parameters__(self): - params = [] - for arg in self.__args__: - # Looks like a genericalias - if hasattr(arg, "__parameters__") and isinstance(arg.__parameters__, tuple): - params.extend(arg.__parameters__) - else: - if _is_typevarlike(arg): - params.append(arg) - return tuple(dict.fromkeys(params)) - def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() @@ -468,57 +456,24 @@ def __getitem__(self, item): # code is copied from typing's _GenericAlias and the builtin # types.GenericAlias. - # A special case in PEP 612 where if X = Callable[P, int], - # then X[int, str] == X[[int, str]]. - param_len = len(self.__parameters__) - if param_len == 0: - raise TypeError(f'{self} is not a generic class') if not isinstance(item, tuple): item = (item,) - if (param_len == 1 and _is_param_expr(self.__parameters__[0]) + # A special case in PEP 612 where if X = Callable[P, int], + # then X[int, str] == X[[int, str]]. + if (len(self.__parameters__) == 1 + and _is_param_expr(self.__parameters__[0]) and item and not _is_param_expr(item[0])): - item = (list(item),) - item_len = len(item) - if item_len != param_len: - raise TypeError(f'Too {"many" if item_len > param_len else "few"}' - f' arguments for {self};' - f' actual {item_len}, expected {param_len}') - subst = dict(zip(self.__parameters__, item)) - new_args = [] - for arg in self.__args__: - if _is_typevarlike(arg): - if _is_param_expr(arg): - arg = subst[arg] - if not _is_param_expr(arg): - raise TypeError(f"Expected a list of types, an ellipsis, " - f"ParamSpec, or Concatenate. Got {arg}") - else: - arg = subst[arg] - # Looks like a GenericAlias - elif hasattr(arg, '__parameters__') and isinstance(arg.__parameters__, tuple): - subparams = arg.__parameters__ - if subparams: - subargs = tuple(subst[x] for x in subparams) - arg = arg[subargs] - if isinstance(arg, tuple): - new_args.extend(arg) - else: - new_args.append(arg) + item = (item,) + + new_args = super().__getitem__(item).__args__ # args[0] occurs due to things like Z[[int, str, bool]] from PEP 612 - if not isinstance(new_args[0], list): + if not isinstance(new_args[0], (tuple, list)): t_result = new_args[-1] t_args = new_args[:-1] new_args = (t_args, t_result) return _CallableGenericAlias(Callable, tuple(new_args)) - -def _is_typevarlike(arg): - obj = type(arg) - # looks like a TypeVar/ParamSpec - return (obj.__module__ == 'typing' - and obj.__name__ in {'ParamSpec', 'TypeVar'}) - def _is_param_expr(obj): """Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or ``_ConcatenateGenericAlias`` from typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 85f74064458f2d..9fd49f25df7d44 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -255,6 +255,36 @@ def test_no_bivariant(self): with self.assertRaises(ValueError): TypeVar('T', covariant=True, contravariant=True) + def test_var_substitution(self): + T = TypeVar('T') + subst = T.__typing_subst__ + self.assertIs(subst(int), int) + self.assertEqual(subst(list[int]), list[int]) + self.assertEqual(subst(List[int]), List[int]) + self.assertEqual(subst(List), List) + self.assertIs(subst(Any), Any) + self.assertIs(subst(None), type(None)) + self.assertIs(subst(T), T) + self.assertEqual(subst(int|str), int|str) + self.assertEqual(subst(Union[int, str]), Union[int, str]) + + def test_bad_var_substitution(self): + T = TypeVar('T') + P = ParamSpec("P") + bad_args = ( + 42, ..., [int], (), (int, str), P, Union, + Generic, Generic[T], Protocol, Protocol[T], + Final, Final[int], ClassVar, ClassVar[int], + ) + for arg in bad_args: + with self.subTest(arg=arg): + with self.assertRaises(TypeError): + T.__typing_subst__(arg) + with self.assertRaises(TypeError): + List[T][arg] + with self.assertRaises(TypeError): + list[T][arg] + class UnionTests(BaseTestCase): @@ -568,8 +598,10 @@ def test_var_substitution(self): C2 = Callable[[KT, T], VT] C3 = Callable[..., T] self.assertEqual(C1[str], Callable[[int, str], str]) + self.assertEqual(C1[None], Callable[[int, type(None)], type(None)]) self.assertEqual(C2[int, float, str], Callable[[int, float], str]) self.assertEqual(C3[int], Callable[..., int]) + self.assertEqual(C3[NoReturn], Callable[..., NoReturn]) # multi chaining C4 = C2[int, VT, str] @@ -2107,7 +2139,10 @@ def test_all_repr_eq_any(self): for obj in objs: self.assertNotEqual(repr(obj), '') self.assertEqual(obj, obj) - if getattr(obj, '__parameters__', None) and len(obj.__parameters__) == 1: + if (getattr(obj, '__parameters__', None) + and not isinstance(obj, typing.TypeVar) + and isinstance(obj.__parameters__, tuple) + and len(obj.__parameters__) == 1): self.assertEqual(obj[Any].__args__, (Any,)) if isinstance(obj, type): for base in obj.__mro__: @@ -4981,21 +5016,29 @@ class X(Generic[P, P2]): self.assertEqual(G1.__args__, ((int, str), (bytes,))) self.assertEqual(G2.__args__, ((int,), (str, bytes))) - def test_no_paramspec_in__parameters__(self): - # ParamSpec should not be found in __parameters__ - # of generics. Usages outside Callable, Concatenate - # and Generic are invalid. + def test_var_substitution(self): T = TypeVar("T") P = ParamSpec("P") - self.assertNotIn(P, List[P].__parameters__) - self.assertIn(T, Tuple[T, P].__parameters__) - - # Test for consistency with builtin generics. - self.assertNotIn(P, list[P].__parameters__) - self.assertIn(T, tuple[T, P].__parameters__) - - self.assertNotIn(P, (list[P] | int).__parameters__) - self.assertIn(T, (tuple[T, P] | int).__parameters__) + subst = P.__typing_subst__ + self.assertEqual(subst((int, str)), (int, str)) + self.assertEqual(subst([int, str]), (int, str)) + self.assertEqual(subst([None]), (type(None),)) + self.assertIs(subst(...), ...) + self.assertIs(subst(P), P) + self.assertEqual(subst(Concatenate[int, P]), Concatenate[int, P]) + + def test_bad_var_substitution(self): + T = TypeVar('T') + P = ParamSpec('P') + bad_args = (42, int, None, T, int|str, Union[int, str]) + for arg in bad_args: + with self.subTest(arg=arg): + with self.assertRaises(TypeError): + P.__typing_subst__(arg) + with self.assertRaises(TypeError): + typing.Callable[P, T][arg, str] + with self.assertRaises(TypeError): + collections.abc.Callable[P, T][arg, str] def test_paramspec_in_nested_generics(self): # Although ParamSpec should not be found in __parameters__ of most @@ -5010,6 +5053,10 @@ def test_paramspec_in_nested_generics(self): self.assertEqual(G1.__parameters__, (P, T)) self.assertEqual(G2.__parameters__, (P, T)) self.assertEqual(G3.__parameters__, (P, T)) + C = Callable[[int, str], float] + self.assertEqual(G1[[int, str], float], List[C]) + self.assertEqual(G2[[int, str], float], list[C]) + self.assertEqual(G3[[int, str], float], list[C] | int) class ConcatenateTests(BaseTestCase): diff --git a/Lib/typing.py b/Lib/typing.py index 0cf9755022e9b8..d515b64a442d55 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -174,11 +174,13 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") - if arg in (Any, NoReturn, ClassVar, Final): + if arg in (Any, NoReturn): + return arg + if allow_special_forms and arg in (ClassVar, Final): return arg if isinstance(arg, _SpecialForm) or arg in (Generic, Protocol): raise TypeError(f"Plain {arg} is not valid as type argument") - if isinstance(arg, (type, TypeVar, ForwardRef, types.UnionType, ParamSpec)): + if isinstance(arg, (type, TypeVar, ForwardRef, types.UnionType)): return arg if not callable(arg): raise TypeError(f"{msg} Got {arg!r:.100}.") @@ -211,21 +213,22 @@ def _type_repr(obj): return repr(obj) -def _collect_type_vars(types_, typevar_types=None): - """Collect all type variable contained - in types in order of first appearance (lexicographic order). For example:: +def _collect_parameters(args): + """Collect all type variables and parameter specifications in args + in order of first appearance (lexicographic order). For example:: - _collect_type_vars((T, List[S, T])) == (T, S) + _collect_parameters((T, Callable[P, T])) == (T, P) """ - if typevar_types is None: - typevar_types = TypeVar - tvars = [] - for t in types_: - if isinstance(t, typevar_types) and t not in tvars: - tvars.append(t) - if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): - tvars.extend([t for t in t.__parameters__ if t not in tvars]) - return tuple(tvars) + parameters = [] + for t in args: + if hasattr(t, '__typing_subst__'): + if t not in parameters: + parameters.append(t) + else: + for x in getattr(t, '__parameters__', ()): + if x not in parameters: + parameters.append(x) + return tuple(parameters) def _check_generic(cls, parameters, elen): @@ -818,6 +821,11 @@ def __init__(self, name, *constraints, bound=None, if def_mod != 'typing': self.__module__ = def_mod + def __typing_subst__(self, arg): + msg = "Parameters to generic types must be types." + arg = _type_check(arg, msg, is_argument=True) + return arg + class ParamSpecArgs(_Final, _Immutable, _root=True): """The args for a ParamSpec object. @@ -918,6 +926,14 @@ def __init__(self, name, *, bound=None, covariant=False, contravariant=False): if def_mod != 'typing': self.__module__ = def_mod + def __typing_subst__(self, arg): + if isinstance(arg, (list, tuple)): + arg = tuple(_type_check(a, "Expected a type.") for a in arg) + elif not _is_param_expr(arg): + raise TypeError(f"Expected a list of types, an ellipsis, " + f"ParamSpec, or Concatenate. Got {arg}") + return arg + def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') @@ -972,7 +988,7 @@ def __getattr__(self, attr): def __setattr__(self, attr, val): if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', - '_typevar_types', '_paramspec_tvars'}: + '_paramspec_tvars'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1001,7 +1017,6 @@ def __dir__(self): class _GenericAlias(_BaseGenericAlias, _root=True): def __init__(self, origin, params, *, inst=True, name=None, - _typevar_types=TypeVar, _paramspec_tvars=False): super().__init__(origin, inst=inst, name=name) if not isinstance(params, tuple): @@ -1009,8 +1024,7 @@ def __init__(self, origin, params, *, inst=True, name=None, self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in params) - self.__parameters__ = _collect_type_vars(params, typevar_types=_typevar_types) - self._typevar_types = _typevar_types + self.__parameters__ = _collect_parameters(params) self._paramspec_tvars = _paramspec_tvars if not name: self.__module__ = origin.__module__ @@ -1047,16 +1061,11 @@ def __getitem__(self, params): subst = dict(zip(self.__parameters__, params)) new_args = [] for arg in self.__args__: - if isinstance(arg, self._typevar_types): - if isinstance(arg, ParamSpec): - arg = subst[arg] - if not _is_param_expr(arg): - raise TypeError(f"Expected a list of types, an ellipsis, " - f"ParamSpec, or Concatenate. Got {arg}") - else: - arg = subst[arg] - elif isinstance(arg, (_GenericAlias, GenericAlias, types.UnionType)): - subparams = arg.__parameters__ + substfunc = getattr(arg, '__typing_subst__', None) + if substfunc: + arg = substfunc(subst[arg]) + else: + subparams = getattr(arg, '__parameters__', ()) if subparams: subargs = tuple(subst[x] for x in subparams) arg = arg[subargs] @@ -1172,7 +1181,6 @@ class _CallableType(_SpecialGenericAlias, _root=True): def copy_with(self, params): return _CallableGenericAlias(self.__origin__, params, name=self._name, inst=self._inst, - _typevar_types=(TypeVar, ParamSpec), _paramspec_tvars=True) def __getitem__(self, params): @@ -1272,7 +1280,6 @@ def __hash__(self): class _ConcatenateGenericAlias(_GenericAlias, _root=True): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs, - _typevar_types=(TypeVar, ParamSpec), _paramspec_tvars=True) def copy_with(self, params): @@ -1333,7 +1340,6 @@ def __class_getitem__(cls, params): else: _check_generic(cls, params, len(cls.__parameters__)) return _GenericAlias(cls, params, - _typevar_types=(TypeVar, ParamSpec), _paramspec_tvars=True) def __init_subclass__(cls, *args, **kwargs): @@ -1346,7 +1352,7 @@ def __init_subclass__(cls, *args, **kwargs): if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _collect_type_vars(cls.__orig_bases__, (TypeVar, ParamSpec)) + tvars = _collect_parameters(cls.__orig_bases__) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index dbe5d89b739629..1d77e74daae6f7 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -156,25 +156,6 @@ ga_repr(PyObject *self) return NULL; } -// isinstance(obj, TypeVar) without importing typing.py. -// Returns -1 for errors. -static int -is_typevar(PyObject *obj) -{ - PyTypeObject *type = Py_TYPE(obj); - if (strcmp(type->tp_name, "TypeVar") != 0) { - return 0; - } - PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); - if (module == NULL) { - return -1; - } - int res = PyUnicode_Check(module) - && _PyUnicode_EqualToASCIIString(module, "typing"); - Py_DECREF(module); - return res; -} - // Index of item in self[:len], or -1 if not found (self is a tuple) static Py_ssize_t tuple_index(PyObject *self, Py_ssize_t len, PyObject *item) @@ -209,13 +190,15 @@ _Py_make_parameters(PyObject *args) Py_ssize_t iparam = 0; for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *t = PyTuple_GET_ITEM(args, iarg); - int typevar = is_typevar(t); - if (typevar < 0) { + _Py_IDENTIFIER(__typing_subst__); + PyObject *subst; + if (_PyObject_LookupAttrId(t, &PyId___typing_subst__, &subst) < 0) { Py_DECREF(parameters); return NULL; } - if (typevar) { + if (subst) { iparam += tuple_add(parameters, iparam, t); + Py_DECREF(subst); } else { _Py_IDENTIFIER(__parameters__); @@ -300,7 +283,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje Py_ssize_t nparams = PyTuple_GET_SIZE(parameters); if (nparams == 0) { return PyErr_Format(PyExc_TypeError, - "There are no type variables left in %R", + "%R is not a generic class", self); } int is_tuple = PyTuple_Check(item); @@ -325,23 +308,24 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(args, iarg); - int typevar = is_typevar(arg); - if (typevar < 0) { + _Py_IDENTIFIER(__typing_subst__); + PyObject *subst; + if (_PyObject_LookupAttrId(arg, &PyId___typing_subst__, &subst) < 0) { Py_DECREF(newargs); return NULL; } - if (typevar) { + if (subst) { Py_ssize_t iparam = tuple_index(parameters, nparams, arg); assert(iparam >= 0); - arg = argitems[iparam]; - Py_INCREF(arg); + arg = PyObject_CallOneArg(subst, argitems[iparam]); + Py_DECREF(subst); } else { arg = subs_tvars(arg, parameters, argitems); - if (arg == NULL) { - Py_DECREF(newargs); - return NULL; - } + } + if (arg == NULL) { + Py_DECREF(newargs); + return NULL; } PyTuple_SET_ITEM(newargs, iarg, arg); }