From 1a970b3c2e3b5f09e7603cbfc220680e3ede0c38 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 16 Apr 2025 06:43:09 -0700 Subject: [PATCH 1/8] gh-132493: Lazily determine protocol attributes gh-132494 made typing.py eagerly import annotationlib again because typing contains several protocols. Avoid this by determining annotations lazily. This should also make protocol creation faster: Unpatched: $ ./python.exe -m timeit -s 'from typing import Protocol, runtime_checkable' '''@runtime_checkable class MyProtocol(Protocol): def meth(self): pass ''' 50000 loops, best of 5: 9.28 usec per loop $ ./python.exe -m timeit -s 'from typing import Protocol, runtime_checkable' '''class MyProtocol(Protocol): def meth(self): pass ''' 50000 loops, best of 5: 9.05 usec per loop Patched: $ ./python.exe -m timeit -s 'from typing import Protocol, runtime_checkable' '''@runtime_checkable class MyProtocol(Protocol): def meth(self): pass ''' 50000 loops, best of 5: 7.69 usec per loop $ ./python.exe -m timeit -s 'from typing import Protocol, runtime_checkable' '''class MyProtocol(Protocol): def meth(self): pass ''' 50000 loops, best of 5: 7.78 usec per loop This was on a debug build though and I haven't compared it with versions where Protocol just accessed `.__annotations__` directly, and it's not a huge difference, so I don't think it's worth calling out the optimization too much. A downside of this change is that any errors that happen during the determination of attributes now happen only the first time isinstance() is called. This seems OK since these errors happen only in fairly exotic circumstances. Another downside is that any attributes added after class initialization now get picked up as protocol members. This came up in the typing test suite due to `@final`, but may cause issues elsewhere too. --- Lib/test/test_typing.py | 11 ++++++--- Lib/typing.py | 55 ++++++++++++++++++++++++----------------- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a252035ed71a03..2b2e911edee47c 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4542,10 +4542,15 @@ class classproperty: def __get__(self, instance, type): raise CustomError + @runtime_checkable + class Commentable(Protocol): + evil = classproperty() + + class Normal: + evil = None + with self.assertRaises(TypeError) as cm: - @runtime_checkable - class Commentable(Protocol): - evil = classproperty() + isinstance(Normal(), Commentable) exc = cm.exception self.assertEqual( diff --git a/Lib/typing.py b/Lib/typing.py index 36789624d2f57a..063f3a65a6a673 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1777,6 +1777,8 @@ class _TypingEllipsis: '__parameters__', '__orig_bases__', '__orig_class__', '_is_protocol', '_is_runtime_protocol', '__protocol_attrs__', '__non_callable_proto_members__', '__type_params__', + '__protocol_attrs_cache__', '__non_callable_proto_members_cache__', + '__final__', }) _SPECIAL_NAMES = frozenset({ @@ -1941,11 +1943,6 @@ def __new__(mcls, name, bases, namespace, /, **kwargs): ) return super().__new__(mcls, name, bases, namespace, **kwargs) - def __init__(cls, *args, **kwargs): - super().__init__(*args, **kwargs) - if getattr(cls, "_is_protocol", False): - cls.__protocol_attrs__ = _get_protocol_attrs(cls) - def __subclasscheck__(cls, other): if cls is Protocol: return type.__subclasscheck__(cls, other) @@ -1997,7 +1994,6 @@ def __instancecheck__(cls, instance): val = getattr_static(instance, attr) except AttributeError: break - # this attribute is set by @runtime_checkable: if val is None and attr not in cls.__non_callable_proto_members__: break else: @@ -2005,6 +2001,37 @@ def __instancecheck__(cls, instance): return False + @property + def __protocol_attrs__(cls): + try: + return cls.__protocol_attrs_cache__ + except AttributeError: + protocol_attrs = _get_protocol_attrs(cls) + cls.__protocol_attrs_cache__ = protocol_attrs + return protocol_attrs + + @property + def __non_callable_proto_members__(cls): + # PEP 544 prohibits using issubclass() + # with protocols that have non-method members. + try: + return cls.__non_callable_proto_members_cache__ + except AttributeError: + non_callable_members = set() + for attr in cls.__protocol_attrs__: + try: + is_callable = callable(getattr(cls, attr, None)) + except Exception as e: + raise TypeError( + f"Failed to determine whether protocol member {attr!r} " + "is a method member" + ) from e + else: + if not is_callable: + non_callable_members.add(attr) + cls.__non_callable_proto_members_cache__ = non_callable_members + return non_callable_members + @classmethod def _proto_hook(cls, other): @@ -2220,22 +2247,6 @@ def close(self): ... raise TypeError('@runtime_checkable can be only applied to protocol classes,' ' got %r' % cls) cls._is_runtime_protocol = True - # PEP 544 prohibits using issubclass() - # with protocols that have non-method members. - # See gh-113320 for why we compute this attribute here, - # rather than in `_ProtocolMeta.__init__` - cls.__non_callable_proto_members__ = set() - for attr in cls.__protocol_attrs__: - try: - is_callable = callable(getattr(cls, attr, None)) - except Exception as e: - raise TypeError( - f"Failed to determine whether protocol member {attr!r} " - "is a method member" - ) from e - else: - if not is_callable: - cls.__non_callable_proto_members__.add(attr) return cls From 781084d17a647141a135f7edad3cedd14107801b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 16 Apr 2025 06:53:42 -0700 Subject: [PATCH 2/8] news --- .../next/Library/2025-04-16-06-50-13.gh-issue-132493.XvnL7t.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-04-16-06-50-13.gh-issue-132493.XvnL7t.rst diff --git a/Misc/NEWS.d/next/Library/2025-04-16-06-50-13.gh-issue-132493.XvnL7t.rst b/Misc/NEWS.d/next/Library/2025-04-16-06-50-13.gh-issue-132493.XvnL7t.rst new file mode 100644 index 00000000000000..1ef92a08ad2762 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-16-06-50-13.gh-issue-132493.XvnL7t.rst @@ -0,0 +1 @@ +Speed up the creation of :class:`typing.Protocol` subclasses. From da69f9d4fbe5275a089fd3344bba5063ec665c70 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 16 Apr 2025 08:26:58 -0700 Subject: [PATCH 3/8] fix with child classes --- Lib/test/test_typing.py | 15 +++++++++++++++ Lib/typing.py | 19 ++++++++++--------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 2b2e911edee47c..d2bf19e95841af 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3132,6 +3132,20 @@ def x(self): ... with self.assertRaisesRegex(TypeError, only_classes_allowed): issubclass(1, BadPG) + def test_lazy_evaluation_with_subprotocols(self): + @runtime_checkable + class Base(Protocol): + x: int + + class Child(Base, Protocol): + y: str + + class Capybara: + x = 43 + + self.assertIsInstance(Capybara(), Base) + self.assertNotIsInstance(Capybara(), Child) + def test_implicit_issubclass_between_two_protocols(self): @runtime_checkable class CallableMembersProto(Protocol): @@ -3826,6 +3840,7 @@ def meth(self): pass '_is_protocol', '_is_runtime_protocol', '__parameters__', '__init__', '__annotations__', '__subclasshook__', '__annotate__', '__annotations_cache__', '__annotate_func__', + '__protocol_attrs_cache__', } self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) self.assertLessEqual( diff --git a/Lib/typing.py b/Lib/typing.py index 063f3a65a6a673..8e286ba0e67789 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1941,7 +1941,9 @@ def __new__(mcls, name, bases, namespace, /, **kwargs): f"Protocols can only inherit from other protocols, " f"got {base!r}" ) - return super().__new__(mcls, name, bases, namespace, **kwargs) + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + cls.__protocol_attrs_cache__ = None + return cls def __subclasscheck__(cls, other): if cls is Protocol: @@ -2003,20 +2005,18 @@ def __instancecheck__(cls, instance): @property def __protocol_attrs__(cls): - try: - return cls.__protocol_attrs_cache__ - except AttributeError: + protocol_attrs = cls.__protocol_attrs_cache__ + if protocol_attrs is None: protocol_attrs = _get_protocol_attrs(cls) cls.__protocol_attrs_cache__ = protocol_attrs - return protocol_attrs + return protocol_attrs @property def __non_callable_proto_members__(cls): # PEP 544 prohibits using issubclass() # with protocols that have non-method members. - try: - return cls.__non_callable_proto_members_cache__ - except AttributeError: + non_callable_members = cls.__non_callable_proto_members_cache__ + if non_callable_members is None: non_callable_members = set() for attr in cls.__protocol_attrs__: try: @@ -2030,7 +2030,7 @@ def __non_callable_proto_members__(cls): if not is_callable: non_callable_members.add(attr) cls.__non_callable_proto_members_cache__ = non_callable_members - return non_callable_members + return non_callable_members @classmethod @@ -2247,6 +2247,7 @@ def close(self): ... raise TypeError('@runtime_checkable can be only applied to protocol classes,' ' got %r' % cls) cls._is_runtime_protocol = True + cls.__non_callable_proto_members_cache__ = None return cls From f10143f5af96397f46ef0b19067208446fc3259b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 16 Apr 2025 09:30:15 -0700 Subject: [PATCH 4/8] review --- Lib/test/test_typing.py | 1 + Lib/typing.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index d2bf19e95841af..ed26e866a28686 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3137,6 +3137,7 @@ def test_lazy_evaluation_with_subprotocols(self): class Base(Protocol): x: int + @runtime_checkable class Child(Base, Protocol): y: str diff --git a/Lib/typing.py b/Lib/typing.py index 8e286ba0e67789..2407076945d4cb 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1925,9 +1925,11 @@ class _ProtocolMeta(ABCMeta): # This metaclass is somewhat unfortunate, # but is necessary for several reasons... def __new__(mcls, name, bases, namespace, /, **kwargs): + is_protocol = False if name == "Protocol" and bases == (Generic,): pass elif Protocol in bases: + is_protocol = True for base in bases: if not ( base in {object, Generic} @@ -1942,7 +1944,8 @@ def __new__(mcls, name, bases, namespace, /, **kwargs): f"got {base!r}" ) cls = super().__new__(mcls, name, bases, namespace, **kwargs) - cls.__protocol_attrs_cache__ = None + if is_protocol: + cls.__protocol_attrs_cache__ = None return cls def __subclasscheck__(cls, other): @@ -2015,7 +2018,7 @@ def __protocol_attrs__(cls): def __non_callable_proto_members__(cls): # PEP 544 prohibits using issubclass() # with protocols that have non-method members. - non_callable_members = cls.__non_callable_proto_members_cache__ + non_callable_members = cls.__non_callable_proto_members_cache__ # set by @runtime_checkable if non_callable_members is None: non_callable_members = set() for attr in cls.__protocol_attrs__: From 61cf996e99e81ef4b0cf547d00a5d73d08fc31c5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 16 Apr 2025 14:08:58 -0700 Subject: [PATCH 5/8] Alternative approach --- Lib/test/test_typing.py | 11 +-- Lib/typing.py | 88 ++++++++----------- ...-04-16-06-50-13.gh-issue-132493.XvnL7t.rst | 1 - 3 files changed, 42 insertions(+), 58 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2025-04-16-06-50-13.gh-issue-132493.XvnL7t.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 3585a1a199f843..df454dd10476ba 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4575,15 +4575,10 @@ class classproperty: def __get__(self, instance, type): raise CustomError - @runtime_checkable - class Commentable(Protocol): - evil = classproperty() - - class Normal: - evil = None - with self.assertRaises(TypeError) as cm: - isinstance(Normal(), Commentable) + @runtime_checkable + class Commentable(Protocol): + evil = classproperty() exc = cm.exception self.assertEqual( diff --git a/Lib/typing.py b/Lib/typing.py index 1cc62e002eb39f..d2b59b1b41cb55 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1777,8 +1777,6 @@ class _TypingEllipsis: '__parameters__', '__orig_bases__', '__orig_class__', '_is_protocol', '_is_runtime_protocol', '__protocol_attrs__', '__non_callable_proto_members__', '__type_params__', - '__protocol_attrs_cache__', '__non_callable_proto_members_cache__', - '__final__', }) _SPECIAL_NAMES = frozenset({ @@ -1803,9 +1801,13 @@ def _get_protocol_attrs(cls): for base in cls.__mro__[:-1]: # without object if base.__name__ in {'Protocol', 'Generic'}: continue - annotations = _lazy_annotationlib.get_annotations( - base, format=_lazy_annotationlib.Format.FORWARDREF - ) + try: + annotations = base.__annotations__ + except Exception: + # Only go through annotationlib to handle deferred annotations if we need to + annotations = _lazy_annotationlib.get_annotations( + base, format=_lazy_annotationlib.Format.FORWARDREF + ) for attr in (*base.__dict__, *annotations): if not attr.startswith('_abc_') and attr not in EXCLUDED_ATTRIBUTES: attrs.add(attr) @@ -1925,11 +1927,9 @@ class _ProtocolMeta(ABCMeta): # This metaclass is somewhat unfortunate, # but is necessary for several reasons... def __new__(mcls, name, bases, namespace, /, **kwargs): - is_protocol = False if name == "Protocol" and bases == (Generic,): pass elif Protocol in bases: - is_protocol = True for base in bases: if not ( base in {object, Generic} @@ -1943,10 +1943,12 @@ def __new__(mcls, name, bases, namespace, /, **kwargs): f"Protocols can only inherit from other protocols, " f"got {base!r}" ) - cls = super().__new__(mcls, name, bases, namespace, **kwargs) - if is_protocol: - cls.__protocol_attrs_cache__ = None - return cls + return super().__new__(mcls, name, bases, namespace, **kwargs) + + def __init__(cls, *args, **kwargs): + super().__init__(*args, **kwargs) + if getattr(cls, "_is_protocol", False): + cls.__protocol_attrs__ = _get_protocol_attrs(cls) def __subclasscheck__(cls, other): if cls is Protocol: @@ -1999,6 +2001,7 @@ def __instancecheck__(cls, instance): val = getattr_static(instance, attr) except AttributeError: break + # this attribute is set by @runtime_checkable: if val is None and attr not in cls.__non_callable_proto_members__: break else: @@ -2006,35 +2009,6 @@ def __instancecheck__(cls, instance): return False - @property - def __protocol_attrs__(cls): - protocol_attrs = cls.__protocol_attrs_cache__ - if protocol_attrs is None: - protocol_attrs = _get_protocol_attrs(cls) - cls.__protocol_attrs_cache__ = protocol_attrs - return protocol_attrs - - @property - def __non_callable_proto_members__(cls): - # PEP 544 prohibits using issubclass() - # with protocols that have non-method members. - non_callable_members = cls.__non_callable_proto_members_cache__ # set by @runtime_checkable - if non_callable_members is None: - non_callable_members = set() - for attr in cls.__protocol_attrs__: - try: - is_callable = callable(getattr(cls, attr, None)) - except Exception as e: - raise TypeError( - f"Failed to determine whether protocol member {attr!r} " - "is a method member" - ) from e - else: - if not is_callable: - non_callable_members.add(attr) - cls.__non_callable_proto_members_cache__ = non_callable_members - return non_callable_members - @classmethod def _proto_hook(cls, other): @@ -2050,14 +2024,15 @@ def _proto_hook(cls, other): break # ...or in annotations, if it is a sub-protocol. - if ( - issubclass(other, Generic) - and getattr(other, "_is_protocol", False) - and attr in _lazy_annotationlib.get_annotations( - base, format=_lazy_annotationlib.Format.FORWARDREF - ) - ): - break + if issubclass(other, Generic) and getattr(other, "_is_protocol", False): + try: + annos = base.__annotations__ + except Exception: + annos = _lazy_annotationlib.get_annotations( + base, format=_lazy_annotationlib.Format.FORWARDREF + ) + if attr in annos: + break else: return NotImplemented return True @@ -2253,7 +2228,22 @@ def close(self): ... raise TypeError('@runtime_checkable can be only applied to protocol classes,' ' got %r' % cls) cls._is_runtime_protocol = True - cls.__non_callable_proto_members_cache__ = None + # PEP 544 prohibits using issubclass() + # with protocols that have non-method members. + # See gh-113320 for why we compute this attribute here, + # rather than in `_ProtocolMeta.__init__` + cls.__non_callable_proto_members__ = set() + for attr in cls.__protocol_attrs__: + try: + is_callable = callable(getattr(cls, attr, None)) + except Exception as e: + raise TypeError( + f"Failed to determine whether protocol member {attr!r} " + "is a method member" + ) from e + else: + if not is_callable: + cls.__non_callable_proto_members__.add(attr) return cls diff --git a/Misc/NEWS.d/next/Library/2025-04-16-06-50-13.gh-issue-132493.XvnL7t.rst b/Misc/NEWS.d/next/Library/2025-04-16-06-50-13.gh-issue-132493.XvnL7t.rst deleted file mode 100644 index 1ef92a08ad2762..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-04-16-06-50-13.gh-issue-132493.XvnL7t.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up the creation of :class:`typing.Protocol` subclasses. From c4f27091d289a2da309f82834f8eca439dafe8b7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 16 Apr 2025 14:54:41 -0700 Subject: [PATCH 6/8] Apply suggestions from code review Co-authored-by: Alex Waygood --- Lib/test/test_typing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index df454dd10476ba..f2622daabfd189 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3149,7 +3149,7 @@ def x(self): ... with self.assertRaisesRegex(TypeError, only_classes_allowed): issubclass(1, BadPG) - def test_lazy_evaluation_with_subprotocols(self): + def test_isinstance_against_superproto_doesnt_affect_subproto_instance(self): @runtime_checkable class Base(Protocol): x: int @@ -3858,7 +3858,6 @@ def meth(self): pass '_is_protocol', '_is_runtime_protocol', '__parameters__', '__init__', '__annotations__', '__subclasshook__', '__annotate__', '__annotations_cache__', '__annotate_func__', - '__protocol_attrs_cache__', } self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) self.assertLessEqual( From 493c8159bd8097bc0fe879e14836bb5f920f6c1f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 16 Apr 2025 20:27:51 -0700 Subject: [PATCH 7/8] Update Lib/typing.py --- Lib/typing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/typing.py b/Lib/typing.py index d2b59b1b41cb55..f70dcd0b5b7b5c 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2025,6 +2025,8 @@ def _proto_hook(cls, other): # ...or in annotations, if it is a sub-protocol. if issubclass(other, Generic) and getattr(other, "_is_protocol", False): + # We avoid the slower path through annotationlib here because in most + # cases it should be unnecessary. try: annos = base.__annotations__ except Exception: From f3ed4fbf1148d54a007dd8e1df811bee6425619d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 17 Apr 2025 08:38:26 -0700 Subject: [PATCH 8/8] annotationlib is now lazily imported again --- Lib/test/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 4f82865c4dd74b..4fd3d53b72c01b 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6338,7 +6338,7 @@ def test_lazy_import(self): "inspect", "re", "contextlib", - # "annotationlib", # TODO + "annotationlib", })