Merge branch 'main' into python_tokenizer

Author: pablogsal

.github/CODEOWNERS

@@ -22,6 +22,7 @@ configure*                    @erlend-aasland @corona10
 **/*hamt*                     @1st1
 Objects/set*                  @rhettinger
 Objects/dict*                 @methane @markshannon
+Objects/typevarobject.c       @JelleZijlstra
 Objects/type*                 @markshannon
 Objects/codeobject.c          @markshannon
 Objects/frameobject.c         @markshannon
@@ -33,6 +34,7 @@ Python/flowgraph.c            @markshannon @iritkatriel
 Python/ast_opt.c              @isidentical
 Lib/test/test_patma.py        @brandtbucher
 Lib/test/test_peepholer.py    @brandtbucher
+Lib/test/test_type_*.py       @JelleZijlstra
 
 # Exceptions
 Lib/traceback.py              @iritkatriel

.github/dependabot.yml

@@ -12,3 +12,10 @@ updates:
         update-types:
           - "version-update:semver-minor"
           - "version-update:semver-patch"
+  - package-ecosystem: "pip"
+    directory: "/Tools/clinic/"
+    schedule:
+      interval: "monthly"
+    labels:
+      - "skip issue"
+      - "skip news"

.github/workflows/build.yml

@@ -36,6 +36,7 @@ jobs:
     timeout-minutes: 10
     outputs:
       run_tests: ${{ steps.check.outputs.run_tests }}
+      run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
     steps:
       - uses: actions/checkout@v3
       - name: Check for source changes
@@ -61,6 +62,17 @@ jobs:
             git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
           fi
 
+          # Check if we should run hypothesis tests
+          GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
+          echo $GIT_BRANCH
+          if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then
+            echo "Branch too old for hypothesis tests"
+            echo "run_hypothesis=false" >> $GITHUB_OUTPUT
+          else
+            echo "Run hypothesis tests"
+            echo "run_hypothesis=true" >> $GITHUB_OUTPUT
+          fi
+
   check_generated_files:
     name: 'Check if generated files are up to date'
     runs-on: ubuntu-latest
@@ -291,6 +303,92 @@ jobs:
     - name: SSL tests
       run: ./python Lib/test/ssltests.py
 
+  test_hypothesis:
+    name: "Hypothesis Tests on Ubuntu"
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
+    env:
+      OPENSSL_VER: 1.1.1t
+      PYTHONSTRICTEXTENSIONBUILD: 1
+    steps:
+    - uses: actions/checkout@v3
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install Dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+    - name: 'Restore OpenSSL build'
+      id: cache-openssl
+      uses: actions/cache@v3
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL
+      if: steps.cache-openssl.outputs.cache-hit != 'true'
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/ccache-action@v1.2
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: Create directories for read-only out-of-tree builds
+      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+    - name: Bind mount sources read-only
+      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Configure CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+    - name: Build CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make -j4
+    - name: Display build info
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make pythoninfo
+    - name: Remount sources writable for tests
+      # some tests write to srcdir, lack of pyc files slows down testing
+      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: "Create hypothesis venv"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        VENV_LOC=$(realpath -m .)/hypovenv
+        VENV_PYTHON=$VENV_LOC/bin/python
+        echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
+        echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
+        ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis
+    - name: "Run tests"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        # Most of the excluded tests are slow test suites with no property tests
+        #
+        # (GH-104097) test_sysconfig is skipped because it has tests that are
+        # failing when executed from inside a virtual environment.
+        ${{ env.VENV_PYTHON }} -m test \
+          -W \
+          -o \
+          -j4 \
+          -x test_asyncio \
+          -x test_multiprocessing_fork \
+          -x test_multiprocessing_forkserver \
+          -x test_multiprocessing_spawn \
+          -x test_concurrent_futures \
+          -x test_socket \
+          -x test_subprocess \
+          -x test_signal \
+          -x test_sysconfig
+
 
   build_asan:
     name: 'Address sanitizer'

.github/workflows/mypy.yml

@@ -0,0 +1,39 @@
+# Workflow to run mypy on select parts of the CPython repo
+name: mypy
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "Tools/clinic/**"
+      - ".github/workflows/mypy.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PIP_DISABLE_PIP_VERSION_CHECK: 1
+  FORCE_COLOR: 1
+  TERM: xterm-256color  # needed for FORCE_COLOR to work on mypy on Ubuntu, see https://github.com/python/mypy/issues/13817
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  mypy:
+    name: Run mypy on Tools/clinic/
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+          cache: pip
+          cache-dependency-path: Tools/clinic/requirements-dev.txt
+      - run: pip install -r Tools/clinic/requirements-dev.txt
+      - run: mypy --config-file Tools/clinic/mypy.ini

Doc/c-api/frame.rst

@@ -130,3 +130,38 @@ See also :ref:`Reflection <reflection>`.
 .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame)
 
    Return the line number that *frame* is currently executing.
+
+
+
+Internal Frames
+---------------
+
+Unless using :pep:`523`, you will not need this.
+
+.. c:struct:: _PyInterpreterFrame
+
+   The interpreter's internal frame representation.
+
+   .. versionadded:: 3.11
+
+.. c:function:: PyObject* PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame);
+
+    Return a :term:`strong reference` to the code object for the frame.
+
+   .. versionadded:: 3.12
+
+
+.. c:function:: int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame);
+
+   Return the byte offset into the last executed instruction.
+
+   .. versionadded:: 3.12
+
+
+.. c:function:: int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame);
+
+   Return the currently executing line number, or -1 if there is no line number.
+
+   .. versionadded:: 3.12
+
+

Doc/howto/clinic.rst

@@ -775,6 +775,9 @@ All Argument Clinic converters accept the following arguments:
     because :pep:`8` mandates that the Python library may not use
     annotations.
 
+  ``unused``
+    Wrap the argument with :c:macro:`Py_UNUSED` in the impl function signature.
+
 In addition, some converters accept additional arguments.  Here is a list
 of these arguments, along with their meanings:
 

Doc/howto/logging.rst

@@ -418,6 +418,7 @@ The flow of log event information in loggers and handlers is illustrated in the
 following diagram.
 
 .. image:: logging_flow.png
+   :class: invert-in-dark-mode
 
 Loggers
 ^^^^^^^

Doc/library/ast.rst

@@ -1724,6 +1724,7 @@ Function and class definitions
             body=[
                 FunctionDef(
                     name='f',
+                    typeparams=[],
                     args=arguments(
                         posonlyargs=[],
                         args=[
@@ -1847,6 +1848,7 @@ Function and class definitions
             body=[
                 ClassDef(
                     name='Foo',
+                    typeparams=[],
                     bases=[
                         Name(id='base1', ctx=Load()),
                         Name(id='base2', ctx=Load())],
@@ -1885,6 +1887,7 @@ Async and await
         body=[
             AsyncFunctionDef(
                 name='f',
+                typeparams=[],
                 args=arguments(
                     posonlyargs=[],
                     args=[],

Doc/library/asyncio-task.rst

@@ -527,6 +527,8 @@ Running Tasks Concurrently
       and there is no running event loop.
 
 
+.. _eager-task-factory:
+
 Eager Task Factory
 ==================
 
@@ -1174,8 +1176,17 @@ Task Object
 
       Return the coroutine object wrapped by the :class:`Task`.
 
+      .. note::
+
+         This will return ``None`` for Tasks which have already
+         completed eagerly. See the :ref:`Eager Task Factory <eager-task-factory>`.
+
       .. versionadded:: 3.8
 
+      .. versionchanged:: 3.12
+
+         Newly added eager task execution means result may be ``None``.
+
    .. method:: get_context()
 
       Return the :class:`contextvars.Context` object

Doc/library/atexit.rst

@@ -20,6 +20,9 @@ at interpreter termination time they will be run in the order ``C``, ``B``,
 program is killed by a signal not handled by Python, when a Python fatal
 internal error is detected, or when :func:`os._exit` is called.
 
+**Note:** The effect of registering or unregistering functions from within
+a cleanup function is undefined.
+
 .. versionchanged:: 3.7
     When used with C-API subinterpreters, registered functions
     are local to the interpreter they were registered in.

Doc/library/dis.rst

@@ -1196,6 +1196,14 @@ iterations of the loop.
 
    .. versionadded:: 3.12
 
+.. opcode:: LOAD_FAST_AND_CLEAR (var_num)
+
+   Pushes a reference to the local ``co_varnames[var_num]`` onto the stack (or
+   pushes ``NULL`` onto the stack if the local variable has not been
+   initialized) and sets ``co_varnames[var_num]`` to ``NULL``.
+
+   .. versionadded:: 3.12
+
 .. opcode:: STORE_FAST (var_num)
 
    Stores ``STACK.pop()`` into the local ``co_varnames[var_num]``.

Doc/library/functions.rst

@@ -168,6 +168,13 @@ are always available.  They are listed here in alphabetical order.
    If :func:`sys.breakpointhook` is not accessible, this function will
    raise :exc:`RuntimeError`.
 
+   By default, the behavior of :func:`breakpoint` can be changed with
+   the :envvar:`PYTHONBREAKPOINT` environment variable.
+   See :func:`sys.breakpointhook` for usage details.
+
+   Note that this is not guaranteed if :func:`sys.breakpointhook`
+   has been replaced.
+
    .. audit-event:: builtins.breakpoint breakpointhook breakpoint
 
    .. versionadded:: 3.7

Doc/library/hashlib.rst

@@ -430,6 +430,7 @@ Constructor functions also accept the following tree hashing parameters:
 
 .. figure:: hashlib-blake2-tree.png
    :alt: Explanation of tree mode parameters.
+   :class: invert-in-dark-mode
 
 See section 2.10 in `BLAKE2 specification
 <https://www.blake2.net/blake2_20130129.pdf>`_ for comprehensive review of tree

Doc/library/http.client.rst

@@ -264,7 +264,10 @@ HTTPConnection Objects
             encode_chunked=False)
 
    This will send a request to the server using the HTTP request
-   method *method* and the selector *url*.
+   method *method* and the request URI *url*. The provided *url* must be
+   an absolute path to conform with :rfc:`RFC 2616 §5.1.2 <2616#section-5.1.2>`
+   (unless connecting to an HTTP proxy server or using the ``OPTIONS`` or
+   ``CONNECT`` methods).
 
    If *body* is specified, the specified data is sent after the headers are
    finished.  It may be a :class:`str`, a :term:`bytes-like object`, an
@@ -279,7 +282,10 @@ HTTPConnection Objects
    iterable are sent as is until the iterable is exhausted.
 
    The *headers* argument should be a mapping of extra HTTP headers to send
-   with the request.
+   with the request. A :rfc:`Host header <2616#section-14.23>`
+   must be provided to conform with :rfc:`RFC 2616 §5.1.2 <2616#section-5.1.2>`
+   (unless connecting to an HTTP proxy server or using the ``OPTIONS`` or
+   ``CONNECT`` methods).
 
    If *headers* contains neither Content-Length nor Transfer-Encoding,
    but there is a request body, one of those
@@ -298,6 +304,16 @@ HTTPConnection Objects
    HTTPConnection object assumes that all encoding is handled by the
    calling code.  If it is ``True``, the body will be chunk-encoded.
 
+   For example, to perform a ``GET`` request to ``https://docs.python.org/3/``::
+
+      >>> import http.client
+      >>> host = "docs.python.org"
+      >>> conn = http.client.HTTPSConnection(host)
+      >>> conn.request("GET", "/3/", headers={"Host": host})
+      >>> response = conn.getresponse()
+      >>> print(response.status, response.reason)
+      200 OK
+
    .. note::
       Chunked transfer encoding has been added to the HTTP protocol
       version 1.1.  Unless the HTTP server is known to handle HTTP 1.1,
@@ -378,6 +394,17 @@ HTTPConnection Objects
       one will be automatically generated and transmitted if not provided in
       the headers argument.
 
+
+.. method:: HTTPConnection.get_proxy_response_headers()
+
+   Returns a dictionary with the headers of the response received from
+   the proxy server to the CONNECT request.
+
+   If the CONNECT request was not sent, the method returns an empty dictionary.
+
+   .. versionadded:: 3.12
+
+
 .. method:: HTTPConnection.connect()
 
    Connect to the server specified when the object was created.  By default,