diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 15a83dd0370e19..531ed060fd3866 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,18 +1,14 @@ variables: - manylinux: false coverage: false -resources: - containers: - - container: manylinux1 - image: pyca/cryptography-manylinux1:x86_64 +trigger: ['master', '3.9', '3.8', '3.7'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 steps: - template: ./prebuild-checks.yml @@ -24,7 +20,7 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 steps: - template: ./docs-steps.yml @@ -35,14 +31,16 @@ jobs: - job: macOS_CI_Tests displayName: macOS CI Tests dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + # bpo-39837: macOS tests on Azure Pipelines are disabled + condition: false variables: testRunTitle: '$(build.sourceBranchName)-macos' testRunPlatform: macos pool: - vmImage: xcode9-macos10.13 + vmImage: macos-10.14 steps: - template: ./macos-steps.yml @@ -54,12 +52,12 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.0j + openssl_version: 1.1.1g steps: - template: ./posix-steps.yml @@ -67,37 +65,6 @@ jobs: dependencies: apt -- job: ManyLinux1_CI_Tests - displayName: ManyLinux1 CI Tests - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['manylinux'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-16.04 - - container: manylinux1 - - variables: - testRunTitle: '$(build.sourceBranchName)-manylinux1' - testRunPlatform: manylinux1 - openssl_version: '' - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: yum - sudo_dependencies: '' - xvfb: false - patchcheck: false - - - job: Ubuntu_Coverage_CI_Tests displayName: Ubuntu CI Tests (coverage) dependsOn: Prebuild @@ -111,12 +78,12 @@ jobs: ) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.0j + openssl_version: 1.1.1g steps: - template: ./posix-steps.yml @@ -131,7 +98,7 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 strategy: matrix: @@ -145,7 +112,7 @@ jobs: buildOpt: '-p x64' testRunTitle: '$(Build.SourceBranchName)-win64' testRunPlatform: win64 - maxParallel: 2 + maxParallel: 4 steps: - template: ./windows-steps.yml diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml index 492e4e34bb2dab..eaf4ad01927aeb 100644 --- a/.azure-pipelines/docs-steps.yml +++ b/.azure-pipelines/docs-steps.yml @@ -12,7 +12,7 @@ steps: inputs: versionSpec: '>=3.6' -- script: python -m pip install sphinx==1.8.2 blurb python-docs-theme +- script: python -m pip install sphinx==2.2.0 blurb python-docs-theme displayName: 'Install build dependencies' - ${{ if ne(parameters.latex, 'true') }}: @@ -21,7 +21,7 @@ steps: displayName: 'Build documentation' - ${{ if eq(parameters.latex, 'true') }}: - - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full + - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full displayName: 'Install LaTeX' - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb' diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml index 647081689454ac..fa38a0df8c87b8 100644 --- a/.azure-pipelines/macos-steps.yml +++ b/.azure-pipelines/macos-steps.yml @@ -6,7 +6,7 @@ steps: - script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev displayName: 'Configure CPython (debug)' -- script: make -s -j4 +- script: make -j4 displayName: 'Build CPython' - script: make pythoninfo @@ -14,6 +14,8 @@ steps: - script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" displayName: 'Tests' + continueOnError: true + timeoutInMinutes: 30 - task: PublishTestResults@2 displayName: 'Publish Test Results' diff --git a/.azure-pipelines/posix-deps-apt.sh b/.azure-pipelines/posix-deps-apt.sh index 4f489903ab569b..e0f4ca5d8d8e88 100755 --- a/.azure-pipelines/posix-deps-apt.sh +++ b/.azure-pipelines/posix-deps-apt.sh @@ -1,3 +1,4 @@ +#!/bin/sh apt-get update apt-get -yq install \ diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml index 3ed3abd02a7146..b6dde593019e6c 100644 --- a/.azure-pipelines/posix-steps.yml +++ b/.azure-pipelines/posix-steps.yml @@ -20,7 +20,7 @@ steps: - script: ./configure --with-pydebug displayName: 'Configure CPython (debug)' -- script: make -s -j4 +- script: make -j4 displayName: 'Build CPython' - ${{ if eq(parameters.coverage, 'true') }}: @@ -49,7 +49,7 @@ steps: - script: ./venv/bin/python -m coverage xml displayName: 'Generate coverage.xml' - - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) + - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml displayName: 'Publish code coverage results' diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 0bd7921bcbefcc..1ffe0a97a2465f 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -1,18 +1,14 @@ variables: - manylinux: false coverage: false -resources: - containers: - - container: manylinux1 - image: pyca/cryptography-manylinux1:x86_64 +pr: ['master', '3.9', '3.8', '3.7'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 steps: - template: ./prebuild-checks.yml @@ -24,7 +20,7 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 steps: - template: ./docs-steps.yml @@ -33,14 +29,16 @@ jobs: - job: macOS_PR_Tests displayName: macOS PR Tests dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + # bpo-39837: macOS tests on Azure Pipelines are disabled + condition: false variables: testRunTitle: '$(system.pullRequest.TargetBranch)-macos' testRunPlatform: macos pool: - vmImage: xcode9-macos10.13 + vmImage: macos-10.14 steps: - template: ./macos-steps.yml @@ -54,12 +52,12 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.0j + openssl_version: 1.1.1g steps: - template: ./posix-steps.yml @@ -67,37 +65,6 @@ jobs: dependencies: apt -- job: ManyLinux1_PR_Tests - displayName: ManyLinux1 PR Tests - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['manylinux'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-16.04 - - container: manylinux1 - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-manylinux1' - testRunPlatform: manylinux1 - openssl_version: '' - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: yum - sudo_dependencies: '' - xvfb: false - patchcheck: false - - - job: Ubuntu_Coverage_PR_Tests displayName: Ubuntu PR Tests (coverage) dependsOn: Prebuild @@ -111,12 +78,12 @@ jobs: ) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.0j + openssl_version: 1.1.1g steps: - template: ./posix-steps.yml @@ -131,7 +98,7 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 strategy: matrix: @@ -145,7 +112,10 @@ jobs: buildOpt: '-p x64' testRunTitle: '$(System.PullRequest.TargetBranch)-win64' testRunPlatform: win64 - maxParallel: 2 + winarm64: + arch: arm64 + buildOpt: '-p arm64' + maxParallel: 4 steps: - template: ./windows-steps.yml diff --git a/.azure-pipelines/windows-release.yml b/.azure-pipelines/windows-release.yml new file mode 100644 index 00000000000000..3d072e3b43e17e --- /dev/null +++ b/.azure-pipelines/windows-release.yml @@ -0,0 +1,129 @@ +name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) + +variables: + __RealSigningCertificate: 'Python Software Foundation' +# QUEUE TIME VARIABLES +# GitRemote: python +# SourceTag: +# DoPGO: true +# SigningCertificate: 'Python Software Foundation' +# SigningDescription: 'Built: $(Build.BuildNumber)' +# DoLayout: true +# DoMSIX: true +# DoNuget: true +# DoEmbed: true +# DoMSI: true +# DoPublish: false +# PyDotOrgUsername: '' +# PyDotOrgServer: '' +# BuildToPublish: '' + +trigger: none +pr: none + +stages: +- stage: Build + displayName: Build binaries + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-build.yml + +- stage: Sign + displayName: Sign binaries + dependsOn: Build + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-sign.yml + +- stage: Layout + displayName: Generate layouts + dependsOn: Sign + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-layout-full.yml + - template: windows-release/stage-layout-embed.yml + - template: windows-release/stage-layout-nuget.yml + +- stage: Pack + dependsOn: Layout + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-pack-nuget.yml + +- stage: Test + dependsOn: Pack + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-test-embed.yml + - template: windows-release/stage-test-nuget.yml + +- stage: Layout_MSIX + displayName: Generate MSIX layouts + dependsOn: Sign + condition: and(succeeded(), and(eq(variables['DoMSIX'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-layout-msix.yml + +- stage: Pack_MSIX + displayName: Package MSIX + dependsOn: Layout_MSIX + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-pack-msix.yml + +- stage: Build_MSI + displayName: Build MSI installer + dependsOn: Sign + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-msi.yml + +- stage: Test_MSI + displayName: Test MSI installer + dependsOn: Build_MSI + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-test-msi.yml + +- stage: PublishPyDotOrg + displayName: Publish to python.org + dependsOn: ['Test_MSI', 'Test'] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishNuget + displayName: Publish to nuget.org + dependsOn: Test + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishStore + displayName: Publish to Store + dependsOn: Pack_MSIX + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-store.yml + + +- stage: PublishExistingPyDotOrg + displayName: Publish existing build to python.org + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishExistingNuget + displayName: Publish existing build to nuget.org + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishExistingStore + displayName: Publish existing build to Store + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-store.yml diff --git a/.azure-pipelines/windows-release/build-steps.yml b/.azure-pipelines/windows-release/build-steps.yml new file mode 100644 index 00000000000000..5ca2016d65f9e1 --- /dev/null +++ b/.azure-pipelines/windows-release/build-steps.yml @@ -0,0 +1,84 @@ +parameters: + ShouldPGO: false + +steps: +- template: ./checkout.yml + +- powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)" + displayName: 'Extract version numbers' + +- ${{ if eq(parameters.ShouldPGO, 'false') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration) + displayName: 'Run build' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- ${{ if eq(parameters.ShouldPGO, 'true') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) --pgo + displayName: 'Run build with PGO' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + displayName: 'Add WinSDK tools to path' + +- powershell: | + $env:SigningCertificate = $null + $(_HostPython) PC\layout -vv -b "$(Build.BinariesDirectory)\bin" -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default --arch $(Arch) + makecat "${env:CAT}.cdf" + del "${env:CAT}.cdf" + if (-not (Test-Path "${env:CAT}.cat")) { + throw "Failed to build catalog file" + } + displayName: 'Generate catalog' + env: + CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python + PYTHON_HEXVERSION: $(VersionHex) + +- task: PublishPipelineArtifact@0 + displayName: 'Publish binaries' + condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))) + inputs: + targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' + artifactName: bin_$(Name) + +- task: PublishPipelineArtifact@0 + displayName: 'Publish binaries for signing' + condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])) + inputs: + targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' + artifactName: unsigned_bin_$(Name) + +- task: CopyFiles@2 + displayName: 'Layout Artifact: symbols' + inputs: + sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch) + targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name) + flatten: true + contents: | + **\*.pdb + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: symbols' + inputs: + PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols' + ArtifactName: symbols diff --git a/.azure-pipelines/windows-release/checkout.yml b/.azure-pipelines/windows-release/checkout.yml new file mode 100644 index 00000000000000..d42d55fff08dda --- /dev/null +++ b/.azure-pipelines/windows-release/checkout.yml @@ -0,0 +1,21 @@ +parameters: + depth: 3 + +steps: +- checkout: none + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/$(SourceTag))' + condition: and(succeeded(), and(variables['GitRemote'], variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone (/$(SourceTag))' + condition: and(succeeded(), and(not(variables['GitRemote']), variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/)' + condition: and(succeeded(), and(variables['GitRemote'], not(variables['SourceTag']))) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone' + condition: and(succeeded(), and(not(variables['GitRemote']), not(variables['SourceTag']))) diff --git a/.azure-pipelines/windows-release/find-sdk.yml b/.azure-pipelines/windows-release/find-sdk.yml new file mode 100644 index 00000000000000..e4de78555b3f66 --- /dev/null +++ b/.azure-pipelines/windows-release/find-sdk.yml @@ -0,0 +1,17 @@ +# Locate the Windows SDK and add its binaries directory to PATH +# +# `toolname` can be overridden to use a different marker file. + +parameters: + toolname: signtool.exe + +steps: + - powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\${{ parameters.toolname }}" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + Write-Host "Adding $($tool.Directory) to PATH" + displayName: 'Add WinSDK tools to path' diff --git a/.azure-pipelines/windows-release/gpg-sign.yml b/.azure-pipelines/windows-release/gpg-sign.yml new file mode 100644 index 00000000000000..0855af8d703df9 --- /dev/null +++ b/.azure-pipelines/windows-release/gpg-sign.yml @@ -0,0 +1,28 @@ +parameters: + GPGKeyFile: $(GPGKey) + GPGPassphrase: $(GPGPassphrase) + Files: '*' + WorkingDirectory: $(Build.BinariesDirectory) + +steps: +- task: DownloadSecureFile@1 + name: gpgkey + inputs: + secureFile: ${{ parameters.GPGKeyFile }} + displayName: 'Download GPG key' + +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch gpg --single-branch --depth 1 --progress -v "gpg" + gpg/gpg2.exe --import "$(gpgkey.secureFilePath)" + (gci -File ${{ parameters.Files }}).FullName | %{ + gpg/gpg2.exe -ba --batch --passphrase ${{ parameters.GPGPassphrase }} $_ + "Made signature for $_" + } + displayName: 'Generate GPG signatures' + workingDirectory: ${{ parameters.WorkingDirectory }} + +- powershell: | + $p = gps "gpg-agent" -EA 0 + if ($p) { $p.Kill() } + displayName: 'Kill GPG agent' + condition: true diff --git a/.azure-pipelines/windows-release/layout-command.yml b/.azure-pipelines/windows-release/layout-command.yml new file mode 100644 index 00000000000000..406ccd859faa6a --- /dev/null +++ b/.azure-pipelines/windows-release/layout-command.yml @@ -0,0 +1,23 @@ +steps: +- task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(HostArch)' + condition: and(succeeded(), variables['HostArch']) + inputs: + artifactName: bin_$(HostArch) + targetPath: $(Build.BinariesDirectory)\bin_$(HostArch) + +- powershell: > + Write-Host ( + '##vso[task.setvariable variable=LayoutCmd]& + "$(Python)" + "{1}\PC\layout" + -vv + --source "{1}" + --build "{0}\bin" + --arch "$(Name)" + --temp "{0}\layout-temp" + --include-cat "{0}\bin\python.cat" + --doc-build "{0}\doc"' + -f ("$(Build.BinariesDirectory)", "$(Build.SourcesDirectory)") + ) + displayName: 'Set LayoutCmd' diff --git a/.azure-pipelines/windows-release/mingw-lib.yml b/.azure-pipelines/windows-release/mingw-lib.yml new file mode 100644 index 00000000000000..30f7d34fa61d23 --- /dev/null +++ b/.azure-pipelines/windows-release/mingw-lib.yml @@ -0,0 +1,13 @@ +parameters: + DllToolOpt: -m i386:x86-64 + #DllToolOpt: -m i386 --as-flags=--32 + +steps: +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch binutils --single-branch --depth 1 --progress -v "binutils" + gci "bin\$(Arch)\python*.dll" | %{ + & "binutils\gendef.exe" $_ | Out-File -Encoding ascii tmp.def + & "binutils\dlltool.exe" --dllname $($_.BaseName).dll --def tmp.def --output-lib "$($_.Directory)\lib$($_.BaseName).a" ${{ parameters.DllToolOpt }} + } + displayName: 'Generate MinGW import library' + workingDirectory: $(Build.BinariesDirectory) diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml new file mode 100644 index 00000000000000..307510a40dd4e5 --- /dev/null +++ b/.azure-pipelines/windows-release/msi-steps.yml @@ -0,0 +1,135 @@ +steps: + - template: ./checkout.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" + displayName: 'Update signing description' + condition: and(succeeded(), not(variables['SigningDescription'])) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + targetPath: $(Build.BinariesDirectory)\doc + + - task: CopyFiles@2 + displayName: 'Merge documentation files' + inputs: + sourceFolder: $(Build.BinariesDirectory)\doc + targetFolder: $(Build.SourcesDirectory)\Doc\build + contents: | + htmlhelp\*.chm + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_win32' + inputs: + artifactName: bin_win32 + targetPath: $(Build.BinariesDirectory)\win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_win32_d' + inputs: + artifactName: bin_win32_d + targetPath: $(Build.BinariesDirectory)\win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_amd64' + inputs: + artifactName: bin_amd64 + targetPath: $(Build.BinariesDirectory)\amd64 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_amd64_d' + inputs: + artifactName: bin_amd64_d + targetPath: $(Build.BinariesDirectory)\amd64 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_win32' + inputs: + artifactName: tcltk_lib_win32 + targetPath: $(Build.BinariesDirectory)\tcltk_lib_win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_amd64' + inputs: + artifactName: tcltk_lib_amd64 + targetPath: $(Build.BinariesDirectory)\tcltk_lib_amd64 + + - powershell: | + copy $(Build.BinariesDirectory)\amd64\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) + + - script: | + call Tools\msi\get_externals.bat + call PCbuild\find_python.bat + echo ##vso[task.setvariable variable=PYTHON]%PYTHON% + call PCbuild/find_msbuild.bat + echo ##vso[task.setvariable variable=MSBUILD]%MSBUILD% + displayName: 'Get external dependencies' + + - script: | + %PYTHON% -m pip install blurb + %PYTHON% -m blurb merge -f Misc\NEWS + displayName: 'Merge NEWS file' + + - script: | + %MSBUILD% Tools\msi\launcher\launcher.wixproj + displayName: 'Build launcher installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false + displayName: 'Build win32 installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\win32\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_win32 + BuildForRelease: true + SuppressMinGWLib: true + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false + displayName: 'Build amd64 installer' + env: + Platform: x64 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_amd64 + BuildForRelease: true + SuppressMinGWLib: true + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (1/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\win32\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\win32 + contents: | + *.msi + *.cab + *.exe + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (2/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\amd64\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\amd64 + contents: | + *.msi + *.cab + *.exe + + - task: PublishPipelineArtifact@0 + displayName: 'Publish MSI' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\msi' + artifactName: msi diff --git a/.azure-pipelines/windows-release/stage-build.yml b/.azure-pipelines/windows-release/stage-build.yml new file mode 100644 index 00000000000000..69f3b1e16451ec --- /dev/null +++ b/.azure-pipelines/windows-release/stage-build.yml @@ -0,0 +1,179 @@ +jobs: +- job: Build_Docs + displayName: Docs build + pool: + name: 'Windows Release' + #vmImage: windows-2019 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: Doc\make.bat html + displayName: 'Build HTML docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + - script: Doc\make.bat htmlhelp + displayName: 'Build CHM docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + #- powershell: | + # mkdir -Force "$(Build.BinariesDirectory)\Doc\htmlhelp" + # iwr "https://www.python.org/ftp/python/3.8.0/python380.chm" -OutFile "$(Build.BinariesDirectory)\Doc\htmlhelp\python390a0.chm" + # displayName: 'Cheat at building CHM docs' + + - task: CopyFiles@2 + displayName: 'Assemble artifact: Doc' + inputs: + sourceFolder: $(Build.BinariesDirectory)\Doc + targetFolder: $(Build.ArtifactStagingDirectory)\Doc + contents: | + html\**\* + htmlhelp\*.chm + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: doc' + inputs: + targetPath: $(Build.ArtifactStagingDirectory)\Doc + artifactName: doc + +- job: Build_Python + displayName: Python build + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Arch: win32 + Platform: x86 + Configuration: Release + _HostPython: .\python + win32_d: + Name: win32_d + Arch: win32 + Platform: x86 + Configuration: Debug + _HostPython: .\python + amd64_d: + Name: amd64_d + Arch: amd64 + Platform: x64 + Configuration: Debug + _HostPython: .\python + arm64: + Name: arm64 + Arch: arm64 + Platform: ARM64 + Configuration: Release + _HostPython: python + arm64_d: + Name: arm64_d + Arch: arm64 + Platform: ARM64 + Configuration: Debug + _HostPython: python + + steps: + - template: ./build-steps.yml + +- job: Build_Python_NonPGO + displayName: Python non-PGO build + condition: and(succeeded(), ne(variables['DoPGO'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + _HostPython: .\python + + steps: + - template: ./build-steps.yml + + +- job: Build_Python_PGO + displayName: Python PGO build + condition: and(succeeded(), eq(variables['DoPGO'], 'true')) + + # Allow up to five hours for PGO + timeoutInMinutes: 300 + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + _HostPython: .\python + + steps: + - template: ./build-steps.yml + parameters: + ShouldPGO: true + + +- job: TclTk_Lib + displayName: Publish Tcl/Tk Library + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: PCbuild\get_externals.bat --no-openssl --no-libffi + displayName: 'Get external dependencies' + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x86 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_win32" + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x64 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_amd64" + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: tcltk_lib_win32' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\tcl_win32' + artifactName: tcltk_lib_win32 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: tcltk_lib_amd64' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\tcl_amd64' + artifactName: tcltk_lib_amd64 diff --git a/.azure-pipelines/windows-release/stage-layout-embed.yml b/.azure-pipelines/windows-release/stage-layout-embed.yml new file mode 100644 index 00000000000000..dbccdead143b21 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-embed.yml @@ -0,0 +1,61 @@ +jobs: +- job: Make_Embed_Layout + displayName: Make embeddable layout + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - template: ./layout-command.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + displayName: 'Extract version numbers' + + - powershell: > + $(LayoutCmd) + --copy "$(Build.ArtifactStagingDirectory)\layout" + --zip "$(Build.ArtifactStagingDirectory)\embed\python-$(VersionText)-embed-$(Name).zip" + --preset-embed + displayName: 'Generate embeddable layout' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_embed_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\layout' + artifactName: layout_embed_$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: embed' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\embed' + ArtifactName: embed diff --git a/.azure-pipelines/windows-release/stage-layout-full.yml b/.azure-pipelines/windows-release/stage-layout-full.yml new file mode 100644 index 00000000000000..8fc8da3e52fe03 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-full.yml @@ -0,0 +1,75 @@ +jobs: +- job: Make_Layouts + displayName: Make layouts + condition: and(succeeded(), eq(variables['DoLayout'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + targetPath: $(Build.BinariesDirectory)\doc + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_$(Name)' + condition: and(succeeded(), variables['TclLibrary']) + inputs: + artifactName: tcltk_lib_$(Name) + targetPath: $(Build.BinariesDirectory)\tcltk_lib + + - powershell: | + copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default + displayName: 'Generate full layout' + env: + TCL_LIBRARY: $(TclLibrary) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_full_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\layout' + artifactName: layout_full_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-msix.yml b/.azure-pipelines/windows-release/stage-layout-msix.yml new file mode 100644 index 00000000000000..def4f7d3c6bee5 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-msix.yml @@ -0,0 +1,97 @@ +jobs: +- job: Make_MSIX_Layout + displayName: Make MSIX layout + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + #win32: + # Name: win32 + # Python: $(Build.BinariesDirectory)\bin\python.exe + # PYTHONHOME: $(Build.SourcesDirectory) + # TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_$(Name)' + condition: and(succeeded(), variables['TclLibrary']) + inputs: + artifactName: tcltk_lib_$(Name) + targetPath: $(Build.BinariesDirectory)\tcltk_lib + + - powershell: | + copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) + + - template: ./layout-command.yml + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx-store" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile + displayName: 'Generate store APPX layout' + env: + TCL_LIBRARY: $(TclLibrary) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_appxstore_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\appx-store' + artifactName: layout_appxstore_$(Name) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: cert' + condition: and(succeeded(), variables['SigningCertificate']) + inputs: + artifactName: cert + targetPath: $(Build.BinariesDirectory)\cert + + - powershell: | + $info = (gc "$(Build.BinariesDirectory)\cert\certinfo.json" | ConvertFrom-JSON) + Write-Host "Side-loadable APPX must be signed with '$($info.Subject)'" + Write-Host "##vso[task.setvariable variable=APPX_DATA_PUBLISHER]$($info.Subject)" + Write-Host "##vso[task.setvariable variable=APPX_DATA_SHA256]$($info.SHA256)" + displayName: 'Override signing parameters' + condition: and(succeeded(), variables['SigningCertificate']) + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests + displayName: 'Generate sideloading APPX layout' + env: + TCL_LIBRARY: $(TclLibrary) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_appx_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\appx' + artifactName: layout_appx_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-nuget.yml b/.azure-pipelines/windows-release/stage-layout-nuget.yml new file mode 100644 index 00000000000000..41cdff850e83be --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-nuget.yml @@ -0,0 +1,52 @@ +jobs: +- job: Make_Nuget_Layout + displayName: Make Nuget layout + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - powershell: | + copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget + displayName: 'Generate nuget layout' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_nuget_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\nuget' + artifactName: layout_nuget_$(Name) diff --git a/.azure-pipelines/windows-release/stage-msi.yml b/.azure-pipelines/windows-release/stage-msi.yml new file mode 100644 index 00000000000000..9b965b09c14748 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-msi.yml @@ -0,0 +1,36 @@ +jobs: +- job: Make_MSI + displayName: Make MSI + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmImage: windows-2019 + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml + +- job: Make_Signed_MSI + displayName: Make signed MSI + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml new file mode 100644 index 00000000000000..26a5712e845ca9 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -0,0 +1,143 @@ +jobs: +- job: Pack_MSIX + displayName: Pack MSIX bundles + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Artifact: appx + Suffix: + ShouldSign: true + amd64_store: + Name: amd64 + Artifact: appxstore + Suffix: -store + Upload: true + arm64: + Name: arm64 + Artifact: appx + Suffix: + ShouldSign: true + arm64_store: + Name: arm64 + Artifact: appxstore + Suffix: -store + Upload: true + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: layout_$(Artifact)_$(Name)' + inputs: + artifactName: layout_$(Artifact)_$(Name) + targetPath: $(Build.BinariesDirectory)\layout + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: symbols' + inputs: + artifactName: symbols + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[task.setvariable variable=Filename]python-$($d.PythonVersion)-$(Name)$(Suffix)" + displayName: 'Extract version numbers' + + - powershell: | + ./Tools/msi/make_appx.ps1 -layout "$(Build.BinariesDirectory)\layout" -msix "$(Build.ArtifactStagingDirectory)\msix\$(Filename).msix" + displayName: 'Build msix' + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msix\$(Filename).appxsym" *.pdb + displayName: 'Build appxsym' + workingDirectory: $(Build.BinariesDirectory)\symbols\$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), or(ne(variables['ShouldSign'], 'true'), not(variables['SigningCertificate']))) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), and(eq(variables['ShouldSign'], 'true'), variables['SigningCertificate'])) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: unsigned_msix + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msixupload\$(Filename).msixupload" * + displayName: 'Build msixupload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + workingDirectory: $(Build.ArtifactStagingDirectory)\msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIXUpload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msixupload' + ArtifactName: msixupload + + +- job: Sign_MSIX + displayName: Sign side-loadable MSIX bundles + dependsOn: + - Pack_MSIX + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + steps: + - template: ./checkout.yml + - template: ./find-sdk.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" + displayName: 'Update signing description' + condition: and(succeeded(), not(variables['SigningDescription'])) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Artifact: unsigned_msix' + inputs: + artifactName: unsigned_msix + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $failed = $true + foreach ($retry in 1..3) { + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix) + if ($?) { + $failed = $false + break + } + sleep 1 + } + if ($failed) { + throw "Failed to sign MSIX" + } + displayName: 'Sign MSIX' + workingDirectory: $(Build.BinariesDirectory)\unsigned_msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\unsigned_msix' + ArtifactName: msix diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml new file mode 100644 index 00000000000000..b100364820d95b --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -0,0 +1,43 @@ +jobs: +- job: Pack_Nuget + displayName: Pack Nuget bundles + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + win32: + Name: win32 + arm64: + Name: arm64 + + steps: + - checkout: none + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: layout_nuget_$(Name)' + inputs: + artifactName: layout_nuget_$(Name) + targetPath: $(Build.BinariesDirectory)\layout + + - task: NugetToolInstaller@0 + displayName: 'Install Nuget' + inputs: + versionSpec: '>=5.0' + + - powershell: | + nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive + displayName: 'Create nuget package' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: nuget' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: nuget diff --git a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml new file mode 100644 index 00000000000000..d5edf44ef5c2ec --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml @@ -0,0 +1,46 @@ +jobs: +- job: Publish_Nuget + displayName: Publish Nuget packages + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - powershell: 'gci pythonarm*.nupkg | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' + displayName: 'Prevent publishing ARM/ARM64 packages' + workingDirectory: '$(Build.BinariesDirectory)\nuget' + condition: and(succeeded(), not(variables['PublishArmPackages'])) + + - task: NuGetCommand@2 + displayName: Push packages + condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + inputs: + command: push + packagesToPush: '$(Build.BinariesDirectory)\nuget\*.nupkg' + nuGetFeedType: external + publishFeedCredentials: 'Python on Nuget' diff --git a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml new file mode 100644 index 00000000000000..4b88bdebf8cc41 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml @@ -0,0 +1,158 @@ +jobs: +- job: Publish_Python + displayName: Publish python.org packages + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true'))) + + pool: + #vmImage: windows-2019 + name: 'Windows Release' + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6 or later' + inputs: + versionSpec: '>=3.6' + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: Doc' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: Doc + targetPath: $(Build.BinariesDirectory)\Doc + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: msi' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact from $(BuildToPublish): Doc' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: Doc + targetPath: $(Build.BinariesDirectory)\Doc + buildType: specific + project: cpython + pipeline: 21 + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact from $(BuildToPublish): msi' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + buildType: specific + project: cpython + pipeline: 21 + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact from $(BuildToPublish): embed' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - powershell: 'gci *embed-arm*.zip | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' + displayName: 'Prevent publishing ARM/ARM64 packages' + workingDirectory: '$(Build.BinariesDirectory)\embed' + condition: and(succeeded(), not(variables['PublishArmPackages'])) + + + - template: ./gpg-sign.yml + parameters: + GPGKeyFile: 'python-signing.key' + Files: 'doc\htmlhelp\*.chm, msi\*\*, embed\*.zip' + + - powershell: > + $(Build.SourcesDirectory)\Tools\msi\uploadrelease.ps1 + -build msi + -user $(PyDotOrgUsername) + -server $(PyDotOrgServer) + -doc_htmlhelp doc\htmlhelp + -embed embed + -skippurge + -skiptest + -skiphash + condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Upload files to python.org' + + - powershell: > + python + "$(Build.SourcesDirectory)\Tools\msi\purge.py" + (gci msi\*\python-*.exe | %{ $_.Name -replace 'python-(.+?)(-|\.exe).+', '$1' } | select -First 1) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Purge CDN' + + - powershell: | + $failures = 0 + gci "msi\*\*-webinstall.exe" -File | %{ + $d = mkdir "tests\$($_.BaseName)" -Force + gci $d -r -File | del + $ic = copy $_ $d -PassThru + "Checking layout for $($ic.Name)" + Start-Process -wait $ic "/passive", "/layout", "$d\layout", "/log", "$d\log\install.log" + if (-not $?) { + Write-Error "Failed to validate layout of $($inst.Name)" + $failures += 1 + } + } + if ($failures) { + Write-Error "Failed to validate $failures installers" + exit 1 + } + #condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Test layouts' + + - powershell: | + $hashes = gci doc\htmlhelp\python*.chm, msi\*\*.exe, embed\*.zip | ` + Sort-Object Name | ` + Format-Table Name, @{ + Label="MD5"; + Expression={(Get-FileHash $_ -Algorithm MD5).Hash} + }, Length -AutoSize | ` + Out-String -Width 4096 + $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force + $hashes | Out-File "$d\hashes.txt" -Encoding ascii + $hashes + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Generate hashes' + + - powershell: | + "Copying:" + (gci msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc).FullName + $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force + move msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc $d -Force + gci msi -Directory | %{ move "msi\$_\*.asc" (mkdir "$d\$_" -Force) } + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Copy GPG signatures for build' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: hashes' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\hashes' + artifactName: hashes diff --git a/.azure-pipelines/windows-release/stage-publish-store.yml b/.azure-pipelines/windows-release/stage-publish-store.yml new file mode 100644 index 00000000000000..e0512b95f27da8 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-store.yml @@ -0,0 +1,35 @@ +jobs: +- job: Publish_Store + displayName: Publish Store packages + condition: and(succeeded(), eq(variables['DoMSIX'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + # TODO: eq(variables['SigningCertificate'], variables['__RealSigningCertificate']) + # If we are not real-signed, DO NOT PUBLISH diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml new file mode 100644 index 00000000000000..584772af8b428e --- /dev/null +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -0,0 +1,130 @@ +parameters: + Include: '*.exe, *.dll, *.pyd, *.cat, *.ps1' + Exclude: 'vcruntime*, libffi*, libcrypto*, libssl*' + +jobs: +- job: Sign_Python + displayName: Sign Python binaries + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + arm64: + Name: arm64 + + steps: + - template: ./checkout.yml + - template: ./find-sdk.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" + displayName: 'Update signing description' + condition: and(succeeded(), not(variables['SigningDescription'])) + + - powershell: | + Write-Host "##vso[build.addbuildtag]signed" + displayName: 'Add build tags' + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: unsigned_bin_$(Name)' + inputs: + artifactName: unsigned_bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - powershell: | + copy "$(Build.SourcesDirectory)\Lib\venv\scripts\common\Activate.ps1" . + displayName: 'Copy files from source' + workingDirectory: $(Build.BinariesDirectory)\bin + + - powershell: | + $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /d "$(SigningDescription)" $files + displayName: 'Sign binaries' + workingDirectory: $(Build.BinariesDirectory)\bin + + - powershell: | + $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) + $failed = $true + foreach ($retry in 1..10) { + signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files + if ($?) { + $failed = $false + break + } + sleep 5 + } + if ($failed) { + Write-Host "##vso[task.logissue type=error]Failed to timestamp files" + } + displayName: 'Timestamp binaries' + workingDirectory: $(Build.BinariesDirectory)\bin + continueOnError: true + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: bin_$(Name)' + inputs: + targetPath: '$(Build.BinariesDirectory)\bin' + artifactName: bin_$(Name) + + +- job: Dump_CertInfo + displayName: Capture certificate info + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + steps: + - checkout: none + + - powershell: | + $m = 'CN=$(SigningCertificate)' + $c = ((gci Cert:\CurrentUser\My), (gci Cert:\LocalMachine\My)) | %{ $_ } | ` + ?{ $_.Subject -match $m } | ` + select -First 1 + if (-not $c) { + Write-Host "Failed to find certificate for $(SigningCertificate)" + exit + } + $d = mkdir "$(Build.BinariesDirectory)\tmp" -Force + $cf = "$d\cert.cer" + [IO.File]::WriteAllBytes($cf, $c.Export("Cer")) + $csha = (certutil -dump $cf | sls "Cert Hash\(sha256\): (.+)").Matches.Groups[1].Value + + $info = @{ Subject=$c.Subject; SHA256=$csha; } + + $d = mkdir "$(Build.BinariesDirectory)\cert" -Force + $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" + displayName: "Extract certificate info" + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: cert' + inputs: + targetPath: '$(Build.BinariesDirectory)\cert' + artifactName: cert + + +- job: Mark_Unsigned + displayName: Tag unsigned build + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmImage: windows-2019 + + steps: + - checkout: none + + - powershell: | + Write-Host "##vso[build.addbuildtag]unsigned" + displayName: 'Add build tag' diff --git a/.azure-pipelines/windows-release/stage-test-embed.yml b/.azure-pipelines/windows-release/stage-test-embed.yml new file mode 100644 index 00000000000000..d99bd74722bacb --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-embed.yml @@ -0,0 +1,41 @@ +jobs: +- job: Test_Embed + displayName: Test Embed + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\embed\python*embed-$(Name).zip" + Expand-Archive -Path $p -DestinationPath "$(Build.BinariesDirectory)\Python" + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Install Python and add to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' diff --git a/.azure-pipelines/windows-release/stage-test-msi.yml b/.azure-pipelines/windows-release/stage-test-msi.yml new file mode 100644 index 00000000000000..21e38c39590f70 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-msi.yml @@ -0,0 +1,108 @@ +jobs: +- job: Test_MSI + displayName: Test MSI + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32_User: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_User + InstallAllUsers: 0 + win32_Machine: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_Machine + InstallAllUsers: 1 + amd64_User: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_User + InstallAllUsers: 0 + amd64_Machine: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_Machine + InstallAllUsers: 1 + + steps: + - checkout: none + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: msi' + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + + - powershell: | + $p = (gci -r *.exe | ?{ $_.Name -match '$(ExeMatch)' } | select -First 1) + Write-Host "##vso[task.setvariable variable=SetupExe]$($p.FullName)" + Write-Host "##vso[task.setvariable variable=SetupExeName]$($p.Name)" + displayName: 'Find installer executable' + workingDirectory: $(Build.BinariesDirectory)\msi + + - script: > + "$(SetupExe)" + /passive + /log "$(Logs)\install\log.txt" + TargetDir="$(Build.BinariesDirectory)\Python" + Include_debug=1 + Include_symbols=1 + InstallAllUsers=$(InstallAllUsers) + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - powershell: | + gci -r "${env:PROGRAMDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-machine Start Menu items' + - powershell: | + gci -r "${env:APPDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-user Start Menu items' + + - powershell: | + gci -r "HKLM:\Software\WOW6432Node\Python" + displayName: 'Capture per-machine 32-bit registry' + - powershell: | + gci -r "HKLM:\Software\Python" + displayName: 'Capture per-machine native registry' + - powershell: | + gci -r "HKCU:\Software\Python" + displayName: 'Capture current-user registry' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' + + - script: | + python -m test -uall -v test_ttk_guionly test_tk test_idle + displayName: 'Test Tkinter and Idle' + + - script: > + "$(SetupExe)" + /passive + /uninstall + /log "$(Logs)\uninstall\log.txt" + displayName: 'Uninstall Python' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: logs' + condition: true + continueOnError: true + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\logs' + ArtifactName: msi_testlogs diff --git a/.azure-pipelines/windows-release/stage-test-nuget.yml b/.azure-pipelines/windows-release/stage-test-nuget.yml new file mode 100644 index 00000000000000..94d815e95226ef --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-nuget.yml @@ -0,0 +1,58 @@ +jobs: +- job: Test_Nuget + displayName: Test Nuget + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Package: pythonx86 + amd64: + Package: python + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: NugetToolInstaller@0 + inputs: + versionSpec: '>= 5' + + - powershell: > + nuget install + $(Package) + -Source "$(Build.BinariesDirectory)\nuget" + -OutputDirectory "$(Build.BinariesDirectory)\install" + -Prerelease + -ExcludeVersion + -NonInteractive + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\install\$(Package)\tools\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml index 794a23a5d77e86..f502c40637c310 100644 --- a/.azure-pipelines/windows-steps.yml +++ b/.azure-pipelines/windows-steps.yml @@ -19,9 +19,11 @@ steps: - script: python.bat -m test.pythoninfo displayName: 'Display build info' + condition: and(succeeded(), variables['testRunPlatform']) - script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir="$(Build.BinariesDirectory)\test" displayName: 'Tests' + condition: and(succeeded(), variables['testRunPlatform']) env: PREFIX: $(Py_OutDir)\$(arch) @@ -32,4 +34,4 @@ steps: mergeTestResults: true testRunTitle: $(testRunTitle) platform: $(testRunPlatform) - condition: succeededOrFailed() + condition: and(succeededOrFailed(), variables['testRunPlatform']) diff --git a/.gitattributes b/.gitattributes index c9a54fbd472ecc..598d8f0f159aee 100644 --- a/.gitattributes +++ b/.gitattributes @@ -41,8 +41,6 @@ PC/readme.txt text eol=crlf # Generated files # https://github.com/github/linguist#generated-code -Include/graminit.h linguist-generated=true -Python/graminit.h linguist-generated=true Modules/clinic/*.h linguist-generated=true Objects/clinic/*.h linguist-generated=true PC/clinic/*.h linguist-generated=true @@ -54,7 +52,7 @@ Python/Python-ast.c linguist-generated=true Include/opcode.h linguist-generated=true Python/opcode_targets.h linguist-generated=true Objects/typeslots.inc linguist-generated=true -Modules/unicodedata_db.h linguist-generated=true +*_db.h linguist-generated=true Doc/library/token-list.inc linguist-generated=true Include/token.h linguist-generated=true Lib/token.py linguist-generated=true diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2ca555b7fa50c4..31003a8ac247e8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,11 +11,26 @@ **/*context* @1st1 **/*genobject* @1st1 **/*hamt* @1st1 +Objects/set* @rhettinger Objects/dict* @methane # Hashing -**/*hashlib* @python/crypto-team -**/*pyhash* @python/crypto-team +**/*hashlib* @python/crypto-team @tiran +**/*pyhash* @python/crypto-team @tiran +**/*sha* @python/crypto-team @tiran +**/*md5* @python/crypto-team @tiran +**/*blake* @python/crypto-team @tiran +/Modules/_blake2/** @python/crypto-team @tiran +/Modules/_sha3/** @python/crypto-team @tiran + +# logging +**/*logging* @vsajip + +# venv +**/*venv* @vsajip + +# Launcher +/PC/launcher.c @vsajip # HTML /Lib/html/ @ezio-melotti @@ -26,15 +41,25 @@ Objects/dict* @methane # Ignoring importlib.h so as to not get flagged on # all pull requests that change the emitted # bytecode. -**/*import*.c @python/import-team -**/*import*.py @python/import-team +**/*import*.c @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw +**/*import*.py @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw # SSL -**/*ssl* @python/crypto-team +**/*ssl* @python/crypto-team @tiran +**/*.pem @python/crypto-team @tiran # CSPRNG -Python/bootstrap_hash.c @python/crypto-team +Python/bootstrap_hash.c @python/crypto-team @tiran + +# Dates and times +**/*datetime* @pganssle @abalkin +**/*str*time* @pganssle @abalkin +Doc/library/time.rst @pganssle @abalkin +Lib/test/test_time.py @pganssle @abalkin +Modules/timemodule.c @pganssle @abalkin +Python/pytime.c @pganssle @abalkin +Include/pytime.h @pganssle @abalkin # Email and related **/*mail* @python/email-team @@ -43,11 +68,22 @@ Python/bootstrap_hash.c @python/crypto-team **/*imap* @python/email-team **/*poplib* @python/email-team +# Garbage collector +/Modules/gcmodule.c @pablogsal +/Doc/library/gc.rst @pablogsal + +# Parser +/Parser/ @pablogsal @lysnikolaou +/Tools/peg_generator/ @pablogsal @lysnikolaou +/Lib/test/test_peg_generator/ @pablogsal @lysnikolaou + # SQLite 3 **/*sqlite* @berkerpeksag # subprocess -**/*subprocess* @gpshead +/Lib/subprocess.py @gpshead +/Lib/test/test_subprocess.py @gpshead +/Modules/*subprocess* @gpshead # Windows /PC/ @python/windows-team @@ -60,6 +96,7 @@ Python/bootstrap_hash.c @python/crypto-team /Tools/msi/ @python/windows-team /Tools/nuget/ @python/windows-team +# Misc **/*itertools* @rhettinger **/*collections* @rhettinger **/*random* @rhettinger @@ -75,6 +112,15 @@ Python/bootstrap_hash.c @python/crypto-team **/*typing* @gvanrossum @ilevkivskyi +**/*asyncore @giampaolo +**/*asynchat @giampaolo +**/*ftplib @giampaolo +**/*shutil @giampaolo + +**/*enum* @ethanfurman +**/*cgi* @ethanfurman +**/*tarfile* @ethanfurman + # macOS /Mac/ @python/macos-team **/*osx_support* @python/macos-team diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 7f912e87084190..a81935d3c9da76 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -8,17 +8,17 @@ Build Status + `Stable buildbots `_ -- 3.7 +- 3.9 - + `Stable buildbots `_ + + `Stable buildbots `_ -- 3.6 +- 3.8 - + `Stable buildbots `_ + + `Stable buildbots `_ -- 2.7 +- 3.7 - + `Stable buildbots `_ + + `Stable buildbots `_ Thank You diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000000000..36c4e2771843e4 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: https://www.python.org/psf/donations/python-dev/ diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 00000000000000..28aea946623cc5 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,18 @@ +# Security Policy + +## Supported Versions + +The Python team applies security fixes according to the table +in [the devguide]( +https://devguide.python.org/#status-of-python-branches +). + +## Reporting a Vulnerability + +Please read the guidelines on reporting security issues [on the +official website]( +https://www.python.org/news/security/#reporting-security-issues-in-python +) for instructions on how to report a security-related problem to +the Python team responsibly. + +To reach the response team, email `security at python dot org`. diff --git a/.github/appveyor.yml b/.github/appveyor.yml deleted file mode 100644 index e8012f69ee5b2e..00000000000000 --- a/.github/appveyor.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: 3.8build{build} -clone_depth: 5 -branches: - only: - - master - - /\d\.\d/ - - buildbot-custom -cache: - - externals -> PCbuild -before_build: - - ps: |+ - if ($env:APPVEYOR_RE_BUILD) { - echo 'Doing full build due to re-build request.' - } elseif (!$env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT) { - echo 'Not a PR, doing full build.' - } else { - git fetch -q origin +refs/heads/$env:APPVEYOR_REPO_BRANCH - $mergebase = git merge-base HEAD FETCH_HEAD - $changes = git diff --name-only HEAD $mergebase | grep -vE '(\.rst$)|(^Doc)|(^Misc)' - If (!$changes) { - echo 'Only docs were updated, stopping build process.' - Exit-AppveyorBuild - } else { - echo 'Doing full build due to non-doc changes in these files:' - echo $changes - } - } - - -build_script: - - cmd: PCbuild\build.bat -e -p x64 - - cmd: PCbuild\amd64\python.exe -m test.pythoninfo -test_script: - - cmd: PCbuild\rt.bat -x64 -q -uall -u-cpu -u-largefile -rwW --slowest --timeout=1200 --fail-env-changed -j0 -environment: - HOST_PYTHON: C:\Python36\python.exe -image: - - Visual Studio 2017 diff --git a/.github/codecov.yml b/.github/codecov.yml index 9d97dfbc43f8d0..ea504f48672eac 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -5,7 +5,7 @@ codecov: comment: off ignore: - "Doc/**/*" - - "Misc/*" + - "Misc/**/*" - "Mac/**/*" - "PC/**/*" - "PCbuild/**/*" diff --git a/.github/problem-matchers/gcc.json b/.github/problem-matchers/gcc.json new file mode 100644 index 00000000000000..bd5ab6c00a7608 --- /dev/null +++ b/.github/problem-matchers/gcc.json @@ -0,0 +1,18 @@ +{ + "__comment": "Taken from vscode-cpptools's Extension/package.json gcc rule", + "problemMatcher": [ + { + "owner": "gcc-problem-matcher", + "pattern": [ + { + "regexp": "^(.*):(\\d+):(\\d+):\\s+(?:fatal\\s+)?(warning|error):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + ] + } + ] +} \ No newline at end of file diff --git a/.github/problem-matchers/sphinx.json b/.github/problem-matchers/sphinx.json new file mode 100644 index 00000000000000..228415f5b7b03f --- /dev/null +++ b/.github/problem-matchers/sphinx.json @@ -0,0 +1,40 @@ +{ + "problemMatcher": [ + { + "owner": "sphinx-problem-matcher", + "pattern": [ + { + "regexp": "^(.*):(\\d+):\\s+(\\w*):\\s+(.*)$", + "file": 1, + "line": 2, + "severity": 3, + "message": 4 + } + ] + }, + { + "owner": "sphinx-problem-matcher-loose", + "pattern": [ + { + "_comment": "A bit of a looser pattern, doesn't look for line numbers, just looks for file names relying on them to start with / and end with .rst", + "regexp": "(\/.*\\.rst):\\s+(\\w*):\\s+(.*)$", + "file": 1, + "severity": 2, + "message": 3 + } + ] + }, + { + "owner": "sphinx-problem-matcher-loose-no-severity", + "pattern": [ + { + "_comment": "Looks for file names ending with .rst and line numbers but without severity", + "regexp": "^(.*\\.rst):(\\d+):(.*)$", + "file": 1, + "line": 2, + "message": 3 + } + ] + } + ] +} \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000000..df68fe271de7a8 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,123 @@ +name: Tests + +# bpo-40548: "paths-ignore" is not used to skip documentation-only PRs, because +# it prevents to mark a job as mandatory. A PR cannot be merged if a job is +# mandatory but not scheduled because of "paths-ignore". +on: + push: + branches: + - master + - 3.9 + - 3.8 + - 3.7 + pull_request: + branches: + - master + - 3.9 + - 3.8 + - 3.7 + +jobs: + check_source: + name: 'Check for source changes' + runs-on: ubuntu-latest + outputs: + run_tests: ${{ steps.check.outputs.run_tests }} + steps: + - uses: actions/checkout@v2 + - name: Check for source changes + id: check + run: | + if [ -z "GITHUB_BASE_REF" ]; then + echo '::set-output name=run_tests::true' + else + git fetch origin $GITHUB_BASE_REF --depth=1 + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more + # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), + # but it requires to download more commits (this job uses + # "git fetch --depth=1"). + # + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git + # 2.26, but Git 2.28 is stricter and fails with "no merge base". + # + # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on + # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF + # into the PR branch anyway. + # + # https://github.com/python/core-workflow/issues/373 + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo '::set-output name=run_tests::true' || true + fi + build_win32: + name: 'Windows (x86)' + runs-on: windows-latest + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v2 + - name: Build CPython + run: .\PCbuild\build.bat -e -p Win32 + - name: Display build info + run: .\python.bat -m test.pythoninfo + - name: Tests + run: .\PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + + build_win_amd64: + name: 'Windows (x64)' + runs-on: windows-latest + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v2 + - name: Build CPython + run: .\PCbuild\build.bat -e -p x64 + - name: Display build info + run: .\python.bat -m test.pythoninfo + - name: Tests + run: .\PCbuild\rt.bat -x64 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + + build_macos: + name: 'macOS' + runs-on: macos-latest + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v2 + - name: Configure CPython + run: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-dev + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: make buildbottest TESTOPTS="-j4 -uall,-cpu" + + build_ubuntu: + name: 'Ubuntu' + runs-on: ubuntu-latest + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + env: + OPENSSL_VER: 1.1.1f + steps: + - uses: actions/checkout@v2 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v1 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $OPENSSL_VER --system Linux + - name: Configure CPython + run: ./configure --with-pydebug --with-openssl=$PWD/multissl/openssl/$OPENSSL_VER + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml new file mode 100644 index 00000000000000..182eb7ce571671 --- /dev/null +++ b/.github/workflows/build_msi.yml @@ -0,0 +1,36 @@ +name: TestsMSI + +on: + push: + branches: + - master + - 3.9 + - 3.8 + - 3.7 + paths: + - 'Tools/msi/**' + pull_request: + branches: + - master + - 3.9 + - 3.8 + - 3.7 + paths: + - 'Tools/msi/**' + +jobs: + build_win32: + name: 'Windows (x86) Installer' + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - name: Build CPython installer + run: .\Tools\msi\build.bat -x86 + + build_win_amd64: + name: 'Windows (x64) Installer' + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - name: Build CPython installer + run: .\Tools\msi\build.bat -x64 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000000000..035348e8a69325 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,94 @@ +name: Coverage + +on: + push: + branches: + - master + - 3.9 + - 3.8 + - 3.7 + paths-ignore: + - 'Doc/**' + - 'Misc/**' + #pull_request: + # branches: + # - master + # - 3.9 + # - 3.8 + # - 3.7 + # paths-ignore: + # - 'Doc/**' + # - 'Misc/**' + +jobs: + coverage_ubuntu: + name: 'Ubuntu (Coverage)' + runs-on: ubuntu-latest + env: + OPENSSL_VER: 1.1.1f + steps: + - uses: actions/checkout@v2 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v1 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $OPENSSL_VER --system Linux + - name: Configure CPython + run: ./configure --with-openssl=$PWD/multissl/openssl/$OPENSSL_VER + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: 'Coverage Preparation' + run: | + ./python -m venv .venv + source ./.venv/bin/activate + python -m pip install -U coverage + python -m pip install -r Misc/requirements-test.txt + python -m test.pythoninfo + export PYTHONPATH=`find .venv -name fullcoverage` + - name: 'Tests with coverage' + run: > + source ./.venv/bin/activate && + xvfb-run python -m coverage + run --branch --pylib + -m test + --fail-env-changed + -uall,-cpu + -x test_multiprocessing_fork + -x test_multiprocessing_forkserver + -x test_multiprocessing_spawn + -x test_concurrent_futures + || true + - name: 'Publish code coverage results' + run: | + export PYTHONPATH= + source ./.venv/bin/activate + bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + c_coverage_ubuntu: + name: 'Ubuntu (C Coverage)' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure CPython + run: ./configure + - name: 'Build CPython and measure coverage' + run: xvfb-run make -j4 coverage-report + - name: 'Publish code coverage results' + if: always() + run: | + make pythoninfo + bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 00000000000000..88c9cbd797ab2d --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,44 @@ +name: Docs + +on: + #push: + # branches: + # - master + # - 3.9 + # - 3.8 + # - 3.7 + # paths: + # - 'Doc/**' + pull_request: + branches: + - master + - 3.9 + - 3.8 + - 3.7 + paths: + - 'Doc/**' + - 'Misc/**' + +jobs: + build_doc: + name: 'Docs' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Register Sphinx problem matcher + run: echo "::add-matcher::.github/problem-matchers/sphinx.json" + - name: 'Install Dependencies' + run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican + - name: 'Configure CPython' + run: ./configure --with-pydebug + - name: 'Build CPython' + run: make -j4 + - name: 'Install build dependencies' + run: make -C Doc/ PYTHON=../python venv + - name: 'Build documentation' + run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest suspicious html + - name: 'Upload' + uses: actions/upload-artifact@v1 + with: + name: doc-html + path: Doc/build/html diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh new file mode 100755 index 00000000000000..2b879d32f8150d --- /dev/null +++ b/.github/workflows/posix-deps-apt.sh @@ -0,0 +1,21 @@ +#!/bin/sh +apt-get update + +apt-get -yq install \ + build-essential \ + zlib1g-dev \ + libbz2-dev \ + liblzma-dev \ + libncurses5-dev \ + libreadline6-dev \ + libsqlite3-dev \ + libssl-dev \ + libgdbm-dev \ + tk-dev \ + lzma \ + lzma-dev \ + liblzma-dev \ + libffi-dev \ + uuid-dev \ + xvfb \ + lcov diff --git a/.gitignore b/.gitignore index 9e60f5a20690bf..0962d3a1841fc4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,15 @@ -# added for local development -.buildaix/ -Modules/python.exp -buildaix/ -installp/ -.gitignore +##### +# First, rules intended to apply in all subdirectories. +# These contain no slash, or only a trailing slash. -# Two-trick pony for OSX and other case insensitive file systems: -# Ignore ./python binary on Unix but still look into ./Python/ directory. -/python -!/Python/ *.cover *.iml *.o +*.a +*.so +*.so.* +*.dylib +*.dll *.orig *.pyc *.pyd @@ -24,6 +22,32 @@ installp/ *.profraw *.dyn .gdb_history +.purify +__pycache__ +.hg/ +.svn/ +.idea/ +tags +TAGS +.vs/ +.vscode/ +gmon.out +.coverage +.mypy_cache/ +.pytest_cache/ + +*.exe +!Lib/distutils/command/*.exe + +# Ignore core dumps... but not Tools/msi/core/ or the like. +core +!core/ + + +##### +# Then, rules meant for a specific location relative to the repo root. +# These must contain a non-trailing slash (and may also have a trailing slash.) + Doc/build/ Doc/venv/ Doc/.venv/ @@ -33,22 +57,21 @@ Include/pydtrace_probes.h Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* -Makefile -Makefile.pre +!Lib/test/data/README +/Makefile +/Makefile.pre Misc/python.pc +Misc/python-embed.pc Misc/python-config.sh -Modules/Setup Modules/Setup.config Modules/Setup.local Modules/config.c Modules/ld_so_aix Programs/_freeze_importlib -Programs/_freeze_importlib.exe Programs/_testembed -Programs/_testembed.exe PC/python_nt*.h PC/pythonnt_rc*.h -PC/*/*.exe +Modules/python.exp PC/*/*.exp PC/*/*.lib PC/*/*.bsc @@ -67,50 +90,35 @@ PCbuild/*-pgi PCbuild/*-pgo PCbuild/*.VC.db PCbuild/*.VC.opendb -PCbuild/.vs/ PCbuild/amd64/ PCbuild/arm32/ +PCbuild/arm64/ PCbuild/obj/ PCbuild/win32/ -.purify -__pycache__ -autom4te.cache -build/ -buildno -config.cache -config.log -config.status -config.status.lineno -core -db_home -.hg/ -.idea/ -ipch/ -libpython*.a -libpython*.so* -libpython*.dylib -libpython*.dll -platform -pybuilddir.txt -pyconfig.h -python-config -python-config.py -python.bat -python.exe -python-gdb.py -python.exe-gdb.py -reflog.txt -.svn/ -tags -TAGS -.coverage -coverage/ -externals/ -htmlcov/ +Tools/unicode/data/ +/autom4te.cache +/build/ +/config.cache +/config.log +/config.status +/config.status.lineno +/platform +/pybuilddir.txt +/pyconfig.h +/python-config +/python-config.py +/python.bat +/python-gdb.py +/python.exe-gdb.py +/reflog.txt +/coverage/ +/externals/ +/htmlcov/ Tools/msi/obj Tools/ssl/amd64 Tools/ssl/win32 -.vs/ -.vscode/ -gmon.out -.mypy_cache/ + +# Two-trick pony for OSX and other case insensitive file systems: +# Ignore ./python binary on Unix but still look into ./Python/ directory. +/python +!/Python/ diff --git a/.travis.yml b/.travis.yml index 6d57ebb1d2fb96..254a4ea35ab944 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: c -dist: xenial -group: beta +dist: bionic # To cache doc-building dependencies and C compiler output. cache: @@ -11,11 +10,10 @@ cache: env: global: - - OPENSSL=1.1.0i + - OPENSSL=1.1.1f - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}" - PATH="${OPENSSL_DIR}/bin:$PATH" - # Use -O3 because we don't use debugger on Travis-CI - - CFLAGS="-I${OPENSSL_DIR}/include -O3" + - CFLAGS="-I${OPENSSL_DIR}/include" - LDFLAGS="-L${OPENSSL_DIR}/lib" # Set rpath with env var instead of -Wl,-rpath linker flag # OpenSSL ignores LDFLAGS when linking bin/openssl @@ -24,7 +22,7 @@ env: branches: only: - master - - /^\d\.\d$/ + - /^\d\.\d+$/ - buildbot-custom matrix: @@ -32,7 +30,8 @@ matrix: allow_failures: - env: OPTIONAL=true include: - - os: linux + - name: "CPython tests" + os: linux language: c compiler: clang # gcc also works, but to keep the # of concurrent builds down, we use one C @@ -42,21 +41,21 @@ matrix: addons: apt: packages: + - gdb - xvfb - - os: linux + - name: "Documentation build" + os: linux language: python # Build the docs against a stable version of Python so code bugs don't hold up doc-related PRs. python: 3.6 env: TESTING=docs before_script: - cd Doc - # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. - # (Updating the version is fine as long as no warnings are raised by doing so.) - # The theme used by the docs is stored separately, so we need to install that as well. - - python -m pip install sphinx==1.8.2 blurb python-docs-theme + - make venv PYTHON=python script: - make check suspicious html SPHINXOPTS="-q -W -j4" - - os: linux + - name: "Documentation tests" + os: linux language: c compiler: clang env: TESTING=doctest @@ -70,39 +69,61 @@ matrix: - make -C Doc/ PYTHON=../python venv script: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest - - os: osx - language: c - compiler: clang - # Testing under macOS is optional until testing stability has been demonstrated. - env: OPTIONAL=true - before_install: - # Python 3 is needed for Argument Clinic and multissl - - HOMEBREW_NO_AUTO_UPDATE=1 brew install xz python3 - - export PATH=$(brew --prefix)/bin:$(brew --prefix)/sbin:$PATH - - os: linux + - name: "Test code coverage (Python)" + os: linux language: c compiler: gcc env: OPTIONAL=true addons: apt: packages: - - lcov - xvfb before_script: + - | + if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] + then + echo "Don't run Python coverage on pull requests." + exit + fi - ./configure - - make coverage -s -j4 + - make -j4 # Need a venv that can parse covered code. - ./python -m venv venv - ./venv/bin/python -m pip install -U coverage + - ./venv/bin/python -m pip install -r Misc/requirements-test.txt - ./venv/bin/python -m test.pythoninfo + - export PYTHONPATH=`find venv -name fullcoverage` script: # Skip tests that re-run the entire test suite. - - xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures + - xvfb-run ./venv/bin/python -m coverage run --branch --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures || true after_script: # Probably should be after_success once test suite updated to run under coverage.py. # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. + - export PYTHONPATH= - source ./venv/bin/activate - - make coverage-lcov - - bash <(curl -s https://codecov.io/bash) + - bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml + - name: "Test code coverage (C)" + os: linux + language: c + compiler: gcc + env: OPTIONAL=true + addons: + apt: + packages: + - lcov + - xvfb + before_script: + - | + if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] + then + echo "Don't run C coverage on pull requests." + exit + fi + - ./configure + script: + - xvfb-run make -j4 coverage-report + after_script: # Probably should be after_success once test suite updated to run under coverage.py. + - make pythoninfo + - bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml before_install: @@ -146,8 +167,11 @@ install: # Travis provides only 2 cores, so don't overdo the parallelism and waste memory. before_script: - - ./configure --with-pydebug - - make -j4 regen-all + # -Og is much faster than -O0 + - CFLAGS="${CFLAGS} -Og" ./configure --with-pydebug + - eval "$(pyenv init -)" + - pyenv global 3.8 + - PYTHON_FOR_REGEN=python3.8 make -j4 regen-all - changes=`git status --porcelain` - | # Check for changes in regenerated files diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index ed15b520548c74..c5f24abe2ca661 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,11 +2,11 @@ Please note that all interactions on [Python Software Foundation](https://www.python.org/psf-landing/)-supported -infrastructure is [covered](https://www.python.org/psf/records/board/minutes/2014-01-06/#management-of-the-psfs-web-properties>) +infrastructure is [covered](https://www.python.org/psf/records/board/minutes/2014-01-06/#management-of-the-psfs-web-properties) by the [PSF Code of Conduct](https://www.python.org/psf/codeofconduct/), -which includes all infrastructure used in the development of Python itself +which includes all the infrastructure used in the development of Python itself (e.g. mailing lists, issue trackers, GitHub, etc.). -In general this means everyone is expected to be open, considerate, and -respectful of others no matter what their position is within the project. +In general, this means that everyone is expected to be **open**, **considerate**, and +**respectful** of others no matter what their position is within the project. diff --git a/Doc/Makefile b/Doc/Makefile index 53877e61329089..f653d70674eb1c 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -13,10 +13,14 @@ SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) SPHINXERRORHANDLING = -W -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \ +# Internal variables. +PAPEROPT_a4 = -D latex_elements.papersize=a4paper +PAPEROPT_letter = -D latex_elements.papersize=letterpaper + +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) \ $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) -.PHONY: help build html htmlhelp latex text changes linkcheck \ +.PHONY: help build html htmlhelp latex text texinfo changes linkcheck \ suspicious coverage doctest pydoc-topics htmlview clean dist check serve \ autobuild-dev autobuild-stable venv @@ -29,6 +33,7 @@ help: @echo " htmlhelp to make HTML files and a HTML help project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " text to make plain text files" + @echo " texinfo to make Texinfo file" @echo " epub to make EPUB files" @echo " changes to make an overview over all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @@ -48,11 +53,19 @@ build: @if [ -f ../Misc/NEWS ] ; then \ echo "Using existing Misc/NEWS file"; \ cp ../Misc/NEWS build/NEWS; \ - elif [ -d ../Misc/NEWS.d ]; then \ - echo "Building NEWS from Misc/NEWS.d with blurb"; \ - $(BLURB) merge -f build/NEWS; \ + elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \ + if [ -d ../Misc/NEWS.d ]; then \ + echo "Building NEWS from Misc/NEWS.d with blurb"; \ + $(BLURB) merge -f build/NEWS; \ + else \ + echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \ + exit 1; \ + fi \ else \ - echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \ + echo ""; \ + echo "Missing the required blurb or sphinx-build tools."; \ + echo "Please run 'make venv' to install local copies."; \ + echo ""; \ exit 1; \ fi $(SPHINXBUILD) $(ALLSPHINXOPTS) @@ -77,6 +90,11 @@ text: BUILDER = text text: build @echo "Build finished; the text files are in build/text." +texinfo: BUILDER = texinfo +texinfo: build + @echo "Build finished; the python.texi file is in build/texinfo." + @echo "Run \`make info' in that directory to run it through makeinfo." + epub: BUILDER = epub epub: build @echo "Build finished; the epub files are in build/epub." @@ -124,7 +142,8 @@ clean: venv: $(PYTHON) -m venv $(VENVDIR) - $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb python-docs-theme + $(VENVDIR)/bin/python3 -m pip install -U pip setuptools + $(VENVDIR)/bin/python3 -m pip install -r requirements.txt @echo "The venv has been created in the $(VENVDIR) directory" dist: @@ -170,6 +189,17 @@ dist: make epub cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub + # archive the texinfo build + rm -rf build/texinfo + make texinfo + make info --directory=build/texinfo + cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo + tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo + bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar + (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo) + rm -r dist/python-$(DISTVERSION)-docs-texinfo + rm dist/python-$(DISTVERSION)-docs-texinfo.tar + check: $(PYTHON) tools/rstlint.py -i tools -i $(VENVDIR) -i README.rst diff --git a/Doc/README.rst b/Doc/README.rst index 31f8a8b7f59328..380ea4fa9b26ad 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -113,6 +113,15 @@ Then, from the ``Doc`` directory, run :: where ```` is one of html, text, latex, or htmlhelp (for explanations see the make targets above). +Deprecation header +================== + +You can define the ``outdated`` variable in ``html_context`` to show a +red banner on each page redirecting to the "latest" version. + +The link points to the same page on ``/3/``, sadly for the moment the +language is lost during the process. + Contributing ============ diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 1e044ad2033d87..a17f04d26fa40b 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -80,7 +80,7 @@ taken on the bug. Article which goes into some detail about how to create a useful bug report. This describes what kind of information is useful and why it is useful. - `Bug Writing Guidelines `_ + `Bug Report Writing Guidelines `_ Information about writing a good bug report. Some of this is specific to the Mozilla project, but describes general good practices. diff --git a/Doc/c-api/abstract.rst b/Doc/c-api/abstract.rst index ad538811127dfb..f5df09fa7fd786 100644 --- a/Doc/c-api/abstract.rst +++ b/Doc/c-api/abstract.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _abstract: @@ -18,9 +18,9 @@ but whose items have not been set to some non-\ ``NULL`` value yet. .. toctree:: object.rst + call.rst number.rst sequence.rst mapping.rst iter.rst buffer.rst - objbuffer.rst diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 25a867f139cc7a..33b0c06a9ebc2e 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _allocating-objects: @@ -48,7 +48,7 @@ Allocating Objects on the Heap improving the memory management efficiency. -.. c:function:: void PyObject_Del(PyObject *op) +.. c:function:: void PyObject_Del(void *op) Releases memory allocated to an object using :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`. This is normally called from the diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 890a0383931933..b8a8f2ff886219 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _apiabiversion: diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index b41130ede416e3..bdaae44e240a02 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _arg-parsing: @@ -50,18 +50,16 @@ Unless otherwise stated, buffers are not NUL-terminated. Some formats require a read-only :term:`bytes-like object`, and set a pointer instead of a buffer structure. They work by checking that -the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is *NULL*, +the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is ``NULL``, which disallows mutable objects such as :class:`bytearray`. .. note:: - For all ``#`` variants of formats (``s#``, ``y#``, etc.), the type of - the length argument (int or :c:type:`Py_ssize_t`) is controlled by - defining the macro :c:macro:`PY_SSIZE_T_CLEAN` before including - :file:`Python.h`. If the macro was defined, length is a - :c:type:`Py_ssize_t` rather than an :c:type:`int`. This behavior will change - in a future Python version to only support :c:type:`Py_ssize_t` and - drop :c:type:`int` support. It is best to always define :c:macro:`PY_SSIZE_T_CLEAN`. + For all ``#`` variants of formats (``s#``, ``y#``, etc.), the macro + :c:macro:`PY_SSIZE_T_CLEAN` must be defined before including + :file:`Python.h`. On Python 3.9 and older, the type of the length argument + is :c:type:`Py_ssize_t` if the :c:macro:`PY_SSIZE_T_CLEAN` macro is defined, + or int otherwise. ``s`` (:class:`str`) [const char \*] @@ -90,7 +88,7 @@ which disallows mutable objects such as :class:`bytearray`. In this case the resulting C string may contain embedded NUL bytes. Unicode objects are converted to C strings using ``'utf-8'`` encoding. -``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, int or :c:type:`Py_ssize_t`] +``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`] Like ``s*``, except that it doesn't accept mutable objects. The result is stored into two C variables, the first one a pointer to a C string, the second one its length. @@ -99,15 +97,15 @@ which disallows mutable objects such as :class:`bytearray`. ``z`` (:class:`str` or ``None``) [const char \*] Like ``s``, but the Python object may also be ``None``, in which case the C - pointer is set to *NULL*. + pointer is set to ``NULL``. ``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] Like ``s*``, but the Python object may also be ``None``, in which case the - ``buf`` member of the :c:type:`Py_buffer` structure is set to *NULL*. + ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``. -``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, int] +``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`] Like ``s#``, but the Python object may also be ``None``, in which case the C - pointer is set to *NULL*. + pointer is set to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] This format converts a bytes-like object to a C pointer to a character @@ -124,19 +122,19 @@ which disallows mutable objects such as :class:`bytearray`. bytes-like objects. **This is the recommended way to accept binary data.** -``y#`` (read-only :term:`bytes-like object`) [const char \*, int] +``y#`` (read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`] This variant on ``s#`` doesn't accept Unicode objects, only bytes-like objects. ``S`` (:class:`bytes`) [PyBytesObject \*] Requires that the Python object is a :class:`bytes` object, without attempting any conversion. Raises :exc:`TypeError` if the object is not - a bytes object. The C variable may also be declared as :c:type:`PyObject\*`. + a bytes object. The C variable may also be declared as :c:type:`PyObject*`. ``Y`` (:class:`bytearray`) [PyByteArrayObject \*] Requires that the Python object is a :class:`bytearray` object, without attempting any conversion. Raises :exc:`TypeError` if the object is not - a :class:`bytearray` object. The C variable may also be declared as :c:type:`PyObject\*`. + a :class:`bytearray` object. The C variable may also be declared as :c:type:`PyObject*`. ``u`` (:class:`str`) [const Py_UNICODE \*] Convert a Python Unicode object to a C pointer to a NUL-terminated buffer of @@ -155,7 +153,7 @@ which disallows mutable objects such as :class:`bytearray`. Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. -``u#`` (:class:`str`) [const Py_UNICODE \*, int] +``u#`` (:class:`str`) [const Py_UNICODE \*, :c:type:`Py_ssize_t`] This variant on ``u`` stores into two C variables, the first one a pointer to a Unicode data buffer, the second one its length. This variant allows null code points. @@ -166,15 +164,15 @@ which disallows mutable objects such as :class:`bytearray`. ``Z`` (:class:`str` or ``None``) [const Py_UNICODE \*] Like ``u``, but the Python object may also be ``None``, in which case the - :c:type:`Py_UNICODE` pointer is set to *NULL*. + :c:type:`Py_UNICODE` pointer is set to ``NULL``. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. -``Z#`` (:class:`str` or ``None``) [const Py_UNICODE \*, int] +``Z#`` (:class:`str` or ``None``) [const Py_UNICODE \*, :c:type:`Py_ssize_t`] Like ``u#``, but the Python object may also be ``None``, in which case the - :c:type:`Py_UNICODE` pointer is set to *NULL*. + :c:type:`Py_UNICODE` pointer is set to ``NULL``. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using @@ -183,7 +181,7 @@ which disallows mutable objects such as :class:`bytearray`. ``U`` (:class:`str`) [PyObject \*] Requires that the Python object is a Unicode object, without attempting any conversion. Raises :exc:`TypeError` if the object is not a Unicode - object. The C variable may also be declared as :c:type:`PyObject\*`. + object. The C variable may also be declared as :c:type:`PyObject*`. ``w*`` (read-write :term:`bytes-like object`) [Py_buffer] This format accepts any object which implements the read-write buffer @@ -196,10 +194,10 @@ which disallows mutable objects such as :class:`bytearray`. It only works for encoded data without embedded NUL bytes. This format requires two arguments. The first is only used as input, and - must be a :c:type:`const char\*` which points to the name of an encoding as a - NUL-terminated string, or *NULL*, in which case ``'utf-8'`` encoding is used. + must be a :c:type:`const char*` which points to the name of an encoding as a + NUL-terminated string, or ``NULL``, in which case ``'utf-8'`` encoding is used. An exception is raised if the named encoding is not known to Python. The - second argument must be a :c:type:`char\*\*`; the value of the pointer it + second argument must be a :c:type:`char**`; the value of the pointer it references will be set to a buffer with the contents of the argument text. The text will be encoded in the encoding specified by the first argument. @@ -213,16 +211,16 @@ which disallows mutable objects such as :class:`bytearray`. recoding them. Instead, the implementation assumes that the byte string object uses the encoding passed in as parameter. -``es#`` (:class:`str`) [const char \*encoding, char \*\*buffer, int \*buffer_length] +``es#`` (:class:`str`) [const char \*encoding, char \*\*buffer, :c:type:`Py_ssize_t` \*buffer_length] This variant on ``s#`` is used for encoding Unicode into a character buffer. Unlike the ``es`` format, this variant allows input data which contains NUL characters. It requires three arguments. The first is only used as input, and must be a - :c:type:`const char\*` which points to the name of an encoding as a - NUL-terminated string, or *NULL*, in which case ``'utf-8'`` encoding is used. + :c:type:`const char*` which points to the name of an encoding as a + NUL-terminated string, or ``NULL``, in which case ``'utf-8'`` encoding is used. An exception is raised if the named encoding is not known to Python. The - second argument must be a :c:type:`char\*\*`; the value of the pointer it + second argument must be a :c:type:`char**`; the value of the pointer it references will be set to a buffer with the contents of the argument text. The text will be encoded in the encoding specified by the first argument. The third argument must be a pointer to an integer; the referenced integer @@ -230,12 +228,12 @@ which disallows mutable objects such as :class:`bytearray`. There are two modes of operation: - If *\*buffer* points a *NULL* pointer, the function will allocate a buffer of + If *\*buffer* points a ``NULL`` pointer, the function will allocate a buffer of the needed size, copy the encoded data into this buffer and set *\*buffer* to reference the newly allocated storage. The caller is responsible for calling :c:func:`PyMem_Free` to free the allocated buffer after usage. - If *\*buffer* points to a non-*NULL* pointer (an already allocated buffer), + If *\*buffer* points to a non-``NULL`` pointer (an already allocated buffer), :c:func:`PyArg_ParseTuple` will use this location as the buffer and interpret the initial value of *\*buffer_length* as the buffer size. It will then copy the encoded data into the buffer and NUL-terminate it. If the buffer is not large @@ -244,7 +242,7 @@ which disallows mutable objects such as :class:`bytearray`. In both cases, *\*buffer_length* is set to the length of the encoded data without the trailing NUL byte. -``et#`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer, int \*buffer_length] +``et#`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer, :c:type:`Py_ssize_t` \*buffer_length] Same as ``es#`` except that byte string objects are passed through without recoding them. Instead, the implementation assumes that the byte string object uses the encoding passed in as parameter. @@ -317,12 +315,12 @@ Other objects ``O`` (object) [PyObject \*] Store a Python object (without any conversion) in a C object pointer. The C program thus receives the actual object that was passed. The object's reference - count is not increased. The pointer stored is not *NULL*. + count is not increased. The pointer stored is not ``NULL``. ``O!`` (object) [*typeobject*, PyObject \*] Store a Python object in a C object pointer. This is similar to ``O``, but takes two C arguments: the first is the address of a Python type object, the - second is the address of the C variable (of type :c:type:`PyObject\*`) into which + second is the address of the C variable (of type :c:type:`PyObject*`) into which the object pointer is stored. If the Python object does not have the required type, :exc:`TypeError` is raised. @@ -331,13 +329,13 @@ Other objects ``O&`` (object) [*converter*, *anything*] Convert a Python object to a C variable through a *converter* function. This takes two arguments: the first is a function, the second is the address of a C - variable (of arbitrary type), converted to :c:type:`void \*`. The *converter* + variable (of arbitrary type), converted to :c:type:`void *`. The *converter* function in turn is called as follows:: status = converter(object, address); where *object* is the Python object to be converted and *address* is the - :c:type:`void\*` argument that was passed to the :c:func:`PyArg_Parse\*` function. + :c:type:`void*` argument that was passed to the :c:func:`PyArg_Parse\*` function. The returned *status* should be ``1`` for a successful conversion and ``0`` if the conversion has failed. When the conversion fails, the *converter* function should raise an exception and leave the content of *address* unmodified. @@ -345,7 +343,7 @@ Other objects If the *converter* returns ``Py_CLEANUP_SUPPORTED``, it may get called a second time if the argument parsing eventually fails, giving the converter a chance to release any memory that it had already allocated. In this second - call, the *object* parameter will be NULL; *address* will have the same value + call, the *object* parameter will be ``NULL``; *address* will have the same value as in the original call. .. versionchanged:: 3.1 @@ -437,7 +435,7 @@ API Functions Parse the parameters of a function that takes both positional and keyword parameters into local variables. The *keywords* argument is a - *NULL*-terminated array of keyword parameter names. Empty names denote + ``NULL``-terminated array of keyword parameter names. Empty names denote :ref:`positional-only parameters `. Returns true on success; on failure, it returns false and raises the appropriate exception. @@ -483,7 +481,7 @@ API Functions *args*; it must actually be a tuple. The length of the tuple must be at least *min* and no more than *max*; *min* and *max* may be equal. Additional arguments must be passed to the function, each of which should be a pointer to a - :c:type:`PyObject\*` variable; these will be filled in with the values from + :c:type:`PyObject*` variable; these will be filled in with the values from *args*; they will contain borrowed references. The variables which correspond to optional parameters not given by *args* will not be filled in; these should be initialized by the caller. This function returns true on success and false if @@ -520,8 +518,8 @@ Building values Create a new value based on a format string similar to those accepted by the :c:func:`PyArg_Parse\*` family of functions and a sequence of values. Returns - the value or *NULL* in the case of an error; an exception will be raised if - *NULL* is returned. + the value or ``NULL`` in the case of an error; an exception will be raised if + ``NULL`` is returned. :c:func:`Py_BuildValue` does not always build a tuple. It builds a tuple only if its format string contains two or more format units. If the format string is @@ -547,41 +545,41 @@ Building values ``s`` (:class:`str` or ``None``) [const char \*] Convert a null-terminated C string to a Python :class:`str` object using ``'utf-8'`` - encoding. If the C string pointer is *NULL*, ``None`` is used. + encoding. If the C string pointer is ``NULL``, ``None`` is used. - ``s#`` (:class:`str` or ``None``) [const char \*, int] + ``s#`` (:class:`str` or ``None``) [const char \*, :c:type:`Py_ssize_t`] Convert a C string and its length to a Python :class:`str` object using ``'utf-8'`` - encoding. If the C string pointer is *NULL*, the length is ignored and + encoding. If the C string pointer is ``NULL``, the length is ignored and ``None`` is returned. ``y`` (:class:`bytes`) [const char \*] This converts a C string to a Python :class:`bytes` object. If the C - string pointer is *NULL*, ``None`` is returned. + string pointer is ``NULL``, ``None`` is returned. - ``y#`` (:class:`bytes`) [const char \*, int] + ``y#`` (:class:`bytes`) [const char \*, :c:type:`Py_ssize_t`] This converts a C string and its lengths to a Python object. If the C - string pointer is *NULL*, ``None`` is returned. + string pointer is ``NULL``, ``None`` is returned. ``z`` (:class:`str` or ``None``) [const char \*] Same as ``s``. - ``z#`` (:class:`str` or ``None``) [const char \*, int] + ``z#`` (:class:`str` or ``None``) [const char \*, :c:type:`Py_ssize_t`] Same as ``s#``. ``u`` (:class:`str`) [const wchar_t \*] Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4) - data to a Python Unicode object. If the Unicode buffer pointer is *NULL*, + data to a Python Unicode object. If the Unicode buffer pointer is ``NULL``, ``None`` is returned. - ``u#`` (:class:`str`) [const wchar_t \*, int] + ``u#`` (:class:`str`) [const wchar_t \*, :c:type:`Py_ssize_t`] Convert a Unicode (UTF-16 or UCS-4) data buffer and its length to a Python - Unicode object. If the Unicode buffer pointer is *NULL*, the length is ignored + Unicode object. If the Unicode buffer pointer is ``NULL``, the length is ignored and ``None`` is returned. ``U`` (:class:`str` or ``None``) [const char \*] Same as ``s``. - ``U#`` (:class:`str` or ``None``) [const char \*, int] + ``U#`` (:class:`str` or ``None``) [const char \*, :c:type:`Py_ssize_t`] Same as ``s#``. ``i`` (:class:`int`) [int] @@ -636,9 +634,9 @@ Building values ``O`` (object) [PyObject \*] Pass a Python object untouched (except for its reference count, which is - incremented by one). If the object passed in is a *NULL* pointer, it is assumed + incremented by one). If the object passed in is a ``NULL`` pointer, it is assumed that this was caused because the call producing the argument found an error and - set an exception. Therefore, :c:func:`Py_BuildValue` will return *NULL* but won't + set an exception. Therefore, :c:func:`Py_BuildValue` will return ``NULL`` but won't raise an exception. If no exception has been raised yet, :exc:`SystemError` is set. @@ -652,8 +650,8 @@ Building values ``O&`` (object) [*converter*, *anything*] Convert *anything* to a Python object through a *converter* function. The - function is called with *anything* (which should be compatible with :c:type:`void - \*`) as its argument and should return a "new" Python object, or *NULL* if an + function is called with *anything* (which should be compatible with :c:type:`void*`) + as its argument and should return a "new" Python object, or ``NULL`` if an error occurred. ``(items)`` (:class:`tuple`) [*matching-items*] @@ -668,7 +666,7 @@ Building values respectively. If there is an error in the format string, the :exc:`SystemError` exception is - set and *NULL* returned. + set and ``NULL`` returned. .. c:function:: PyObject* Py_VaBuildValue(const char *format, va_list vargs) diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst index a9fb342f7c0f8e..ce8de6e22f44ca 100644 --- a/Doc/c-api/bool.rst +++ b/Doc/c-api/bool.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _boolobjects: diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index c7c1e3cc745ac4..e32719373cc716 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. index:: single: buffer protocol @@ -89,7 +89,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:type:: Py_buffer - .. c:member:: void \*buf + .. c:member:: void *buf A pointer to the start of the logical structure described by the buffer fields. This can be any location within the underlying physical memory @@ -99,16 +99,16 @@ a buffer, see :c:func:`PyObject_GetBuffer`. For :term:`contiguous` arrays, the value points to the beginning of the memory block. - .. c:member:: void \*obj + .. c:member:: void *obj A new reference to the exporting object. The reference is owned by - the consumer and automatically decremented and set to *NULL* by + the consumer and automatically decremented and set to ``NULL`` by :c:func:`PyBuffer_Release`. The field is the equivalent of the return value of any standard C-API function. As a special case, for *temporary* buffers that are wrapped by :c:func:`PyMemoryView_FromBuffer` or :c:func:`PyBuffer_FillInfo` - this field is *NULL*. In general, exporting objects MUST NOT + this field is ``NULL``. In general, exporting objects MUST NOT use this scheme. .. c:member:: Py_ssize_t len @@ -130,25 +130,25 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:member:: Py_ssize_t itemsize Item size in bytes of a single element. Same as the value of :func:`struct.calcsize` - called on non-NULL :c:member:`~Py_buffer.format` values. + called on non-``NULL`` :c:member:`~Py_buffer.format` values. Important exception: If a consumer requests a buffer without the :c:macro:`PyBUF_FORMAT` flag, :c:member:`~Py_buffer.format` will - be set to *NULL*, but :c:member:`~Py_buffer.itemsize` still has + be set to ``NULL``, but :c:member:`~Py_buffer.itemsize` still has the value for the original format. If :c:member:`~Py_buffer.shape` is present, the equality ``product(shape) * itemsize == len`` still holds and the consumer can use :c:member:`~Py_buffer.itemsize` to navigate the buffer. - If :c:member:`~Py_buffer.shape` is *NULL* as a result of a :c:macro:`PyBUF_SIMPLE` + If :c:member:`~Py_buffer.shape` is ``NULL`` as a result of a :c:macro:`PyBUF_SIMPLE` or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - .. c:member:: const char \*format + .. c:member:: const char *format A *NUL* terminated string in :mod:`struct` module style syntax describing - the contents of a single item. If this is *NULL*, ``"B"`` (unsigned bytes) + the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes) is assumed. This field is controlled by the :c:macro:`PyBUF_FORMAT` flag. @@ -158,13 +158,13 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The number of dimensions the memory represents as an n-dimensional array. If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` - and :c:member:`~Py_buffer.suboffsets` MUST be *NULL*. + and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions to 64. Exporters MUST respect this limit, consumers of multi-dimensional buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions. - .. c:member:: Py_ssize_t \*shape + .. c:member:: Py_ssize_t *shape An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` indicating the shape of the memory as an n-dimensional array. Note that @@ -177,7 +177,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The shape array is read-only for the consumer. - .. c:member:: Py_ssize_t \*strides + .. c:member:: Py_ssize_t *strides An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` giving the number of bytes to skip to get to a new element in each @@ -189,7 +189,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The strides array is read-only for the consumer. - .. c:member:: Py_ssize_t \*suboffsets + .. c:member:: Py_ssize_t *suboffsets An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim`. If ``suboffsets[n] >= 0``, the values stored along the nth dimension are @@ -199,7 +199,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. memory block). If all suboffsets are negative (i.e. no de-referencing is needed), then - this field must be NULL (the default value). + this field must be ``NULL`` (the default value). This type of array representation is used by the Python Imaging Library (PIL). See `complex arrays`_ for further information how to access elements @@ -207,7 +207,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The suboffsets array is read-only for the consumer. - .. c:member:: void \*internal + .. c:member:: void *internal This is for use internally by the exporting object. For example, this might be re-cast as an integer by the exporter and used to store flags @@ -248,7 +248,7 @@ readonly, format .. c:macro:: PyBUF_FORMAT Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST - be filled in correctly. Otherwise, this field MUST be *NULL*. + be filled in correctly. Otherwise, this field MUST be ``NULL``. :c:macro:`PyBUF_WRITABLE` can be \|'d to any of the flags in the next section. @@ -301,7 +301,7 @@ must be C-contiguous. +-----------------------------------+-------+---------+------------+--------+ | .. c:macro:: PyBUF_ANY_CONTIGUOUS | yes | yes | NULL | C or F | +-----------------------------------+-------+---------+------------+--------+ -| .. c:macro:: PyBUF_ND | yes | NULL | NULL | C | +| :c:macro:`PyBUF_ND` | yes | NULL | NULL | C | +-----------------------------------+-------+---------+------------+--------+ @@ -349,14 +349,16 @@ The logical structure of NumPy-style arrays is defined by :c:member:`~Py_buffer. If ``ndim == 0``, the memory location pointed to by :c:member:`~Py_buffer.buf` is interpreted as a scalar of size :c:member:`~Py_buffer.itemsize`. In that case, -both :c:member:`~Py_buffer.shape` and :c:member:`~Py_buffer.strides` are *NULL*. +both :c:member:`~Py_buffer.shape` and :c:member:`~Py_buffer.strides` are ``NULL``. -If :c:member:`~Py_buffer.strides` is *NULL*, the array is interpreted as +If :c:member:`~Py_buffer.strides` is ``NULL``, the array is interpreted as a standard n-dimensional C-array. Otherwise, the consumer must access an n-dimensional array as follows: - ``ptr = (char *)buf + indices[0] * strides[0] + ... + indices[n-1] * strides[n-1]`` - ``item = *((typeof(item) *)ptr);`` +.. code-block:: c + + ptr = (char *)buf + indices[0] * strides[0] + ... + indices[n-1] * strides[n-1]; + item = *((typeof(item) *)ptr); As noted above, :c:member:`~Py_buffer.buf` can point to any location within @@ -405,7 +407,7 @@ to two ``char x[2][3]`` arrays that can be located anywhere in memory. Here is a function that returns a pointer to the element in an N-D array -pointed to by an N-dimensional index when there are both non-NULL strides +pointed to by an N-dimensional index when there are both non-``NULL`` strides and suboffsets:: void *get_item_pointer(int ndim, void *buf, Py_ssize_t *strides, @@ -436,12 +438,12 @@ Buffer-related functions Send a request to *exporter* to fill in *view* as specified by *flags*. If the exporter cannot provide a buffer of the exact type, it MUST raise - :c:data:`PyExc_BufferError`, set :c:member:`view->obj` to *NULL* and + :c:data:`PyExc_BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``. - On success, fill in *view*, set :c:member:`view->obj` to a new reference + On success, fill in *view*, set ``view->obj`` to a new reference to *exporter* and return 0. In the case of chained buffer providers - that redirect requests to a single object, :c:member:`view->obj` MAY + that redirect requests to a single object, ``view->obj`` MAY refer to this object instead of *exporter* (See :ref:`Buffer Object Structures `). Successful calls to :c:func:`PyObject_GetBuffer` must be paired with calls @@ -453,17 +455,19 @@ Buffer-related functions .. c:function:: void PyBuffer_Release(Py_buffer *view) Release the buffer *view* and decrement the reference count for - :c:member:`view->obj`. This function MUST be called when the buffer + ``view->obj``. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur. It is an error to call this function on a buffer that was not obtained via :c:func:`PyObject_GetBuffer`. -.. c:function:: Py_ssize_t PyBuffer_SizeFromFormat(const char *) +.. c:function:: Py_ssize_t PyBuffer_SizeFromFormat(const char *format) Return the implied :c:data:`~Py_buffer.itemsize` from :c:data:`~Py_buffer.format`. - This function is not yet implemented. + On error, raise an exception and return -1. + + .. versionadded:: 3.9 .. c:function:: int PyBuffer_IsContiguous(Py_buffer *view, char order) @@ -473,11 +477,24 @@ Buffer-related functions (*order* is ``'A'``). Return ``0`` otherwise. This function always succeeds. +.. c:function:: void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices) + + Get the memory area pointed to by the *indices* inside the given *view*. + *indices* must point to an array of ``view->ndim`` indices. + + +.. c:function:: int PyBuffer_FromContiguous(Py_buffer *view, void *buf, Py_ssize_t len, char fort) + + Copy contiguous *len* bytes from *buf* to *view*. + *fort* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). + ``0`` is returned on success, ``-1`` on error. + + .. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order) Copy *len* bytes from *src* to its contiguous representation in *buf*. - *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). - ``0`` is returned on success, ``-1`` on error. + *order* can be ``'C'`` or ``'F'`` or ``'A'`` (for C-style or Fortran-style + ordering or either one). ``0`` is returned on success, ``-1`` on error. This function fails if *len* != *src->len*. @@ -499,10 +516,10 @@ Buffer-related functions *view* as specified by flags, unless *buf* has been designated as read-only and :c:macro:`PyBUF_WRITABLE` is set in *flags*. - On success, set :c:member:`view->obj` to a new reference to *exporter* and + On success, set ``view->obj`` to a new reference to *exporter* and return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set - :c:member:`view->obj` to *NULL* and return ``-1``; + ``view->obj`` to ``NULL`` and return ``-1``; If this function is used as part of a :ref:`getbufferproc `, *exporter* MUST be set to the exporting object and *flags* must be passed - unmodified. Otherwise, *exporter* MUST be NULL. + unmodified. Otherwise, *exporter* MUST be ``NULL``. diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 41b6e3c71be53d..b2f409c15abb7a 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _bytearrayobjects: @@ -48,7 +48,7 @@ Direct API functions .. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) Create a new bytearray object from *string* and its length, *len*. On - failure, *NULL* is returned. + failure, ``NULL`` is returned. .. c:function:: PyObject* PyByteArray_Concat(PyObject *a, PyObject *b) @@ -58,13 +58,13 @@ Direct API functions .. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) - Return the size of *bytearray* after checking for a *NULL* pointer. + Return the size of *bytearray* after checking for a ``NULL`` pointer. .. c:function:: char* PyByteArray_AsString(PyObject *bytearray) Return the contents of *bytearray* as a char array after checking for a - *NULL* pointer. The returned array always has an extra + ``NULL`` pointer. The returned array always has an extra null byte appended. diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 5b9ebf6b6af5f5..0e33ed2c7c9403 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _bytesobjects: @@ -37,14 +37,14 @@ called with a non-bytes parameter. .. c:function:: PyObject* PyBytes_FromString(const char *v) Return a new bytes object with a copy of the string *v* as value on success, - and *NULL* on failure. The parameter *v* must not be *NULL*; it will not be + and ``NULL`` on failure. The parameter *v* must not be ``NULL``; it will not be checked. .. c:function:: PyObject* PyBytes_FromStringAndSize(const char *v, Py_ssize_t len) Return a new bytes object with a copy of the string *v* as value and length - *len* on success, and *NULL* on failure. If *v* is *NULL*, the contents of + *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. @@ -145,7 +145,7 @@ called with a non-bytes parameter. whether there are any other null bytes. The data must not be modified in any way, unless the object was just created using ``PyBytes_FromStringAndSize(NULL, size)``. It must not be deallocated. If - *o* is not a bytes object at all, :c:func:`PyBytes_AsString` returns *NULL* + *o* is not a bytes object at all, :c:func:`PyBytes_AsString` returns ``NULL`` and raises :exc:`TypeError`. @@ -159,7 +159,7 @@ called with a non-bytes parameter. Return the null-terminated contents of the object *obj* through the output variables *buffer* and *length*. - If *length* is *NULL*, the bytes object + If *length* is ``NULL``, the bytes object may not contain embedded null bytes; if it does, the function returns ``-1`` and a :exc:`ValueError` is raised. @@ -181,7 +181,7 @@ called with a non-bytes parameter. appended to *bytes*; the caller will own the new reference. The reference to the old value of *bytes* will be stolen. If the new object cannot be created, the old reference to *bytes* will still be discarded and the value - of *\*bytes* will be set to *NULL*; the appropriate exception will be set. + of *\*bytes* will be set to ``NULL``; the appropriate exception will be set. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) @@ -201,5 +201,5 @@ called with a non-bytes parameter. desired. On success, *\*bytes* holds the resized bytes object and ``0`` is returned; the address in *\*bytes* may differ from its input value. If the reallocation fails, the original bytes object at *\*bytes* is deallocated, - *\*bytes* is set to *NULL*, :exc:`MemoryError` is set, and ``-1`` is + *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst new file mode 100644 index 00000000000000..31dc9c8031fdb6 --- /dev/null +++ b/Doc/c-api/call.rst @@ -0,0 +1,424 @@ +.. highlight:: c + +.. _call: + +Call Protocol +============= + +CPython supports two different calling protocols: +*tp_call* and vectorcall. + +The *tp_call* Protocol +---------------------- + +Instances of classes that set :c:member:`~PyTypeObject.tp_call` are callable. +The signature of the slot is:: + + PyObject *tp_call(PyObject *callable, PyObject *args, PyObject *kwargs); + +A call is made using a tuple for the positional arguments +and a dict for the keyword arguments, similarly to +``callable(*args, **kwargs)`` in Python code. +*args* must be non-NULL (use an empty tuple if there are no arguments) +but *kwargs* may be *NULL* if there are no keyword arguments. + +This convention is not only used by *tp_call*: +:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_init` +also pass arguments this way. + +To call an object, use :c:func:`PyObject_Call` or other +:ref:`call API `. + + +.. _vectorcall: + +The Vectorcall Protocol +----------------------- + +.. versionadded:: 3.9 + +The vectorcall protocol was introduced in :pep:`590` as an additional protocol +for making calls more efficient. + +As rule of thumb, CPython will prefer the vectorcall for internal calls +if the callable supports it. However, this is not a hard rule. +Additionally, some third-party extensions use *tp_call* directly +(rather than using :c:func:`PyObject_Call`). +Therefore, a class supporting vectorcall must also implement +:c:member:`~PyTypeObject.tp_call`. +Moreover, the callable must behave the same +regardless of which protocol is used. +The recommended way to achieve this is by setting +:c:member:`~PyTypeObject.tp_call` to :c:func:`PyVectorcall_Call`. +This bears repeating: + +.. warning:: + + A class supporting vectorcall **must** also implement + :c:member:`~PyTypeObject.tp_call` with the same semantics. + +A class should not implement vectorcall if that would be slower +than *tp_call*. For example, if the callee needs to convert +the arguments to an args tuple and kwargs dict anyway, then there is no point +in implementing vectorcall. + +Classes can implement the vectorcall protocol by enabling the +:const:`Py_TPFLAGS_HAVE_VECTORCALL` flag and setting +:c:member:`~PyTypeObject.tp_vectorcall_offset` to the offset inside the +object structure where a *vectorcallfunc* appears. +This is a pointer to a function with the following signature: + +.. c:type:: PyObject *(*vectorcallfunc)(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + +- *callable* is the object being called. +- *args* is a C array consisting of the positional arguments followed by the + values of the keyword arguments. + This can be *NULL* if there are no arguments. +- *nargsf* is the number of positional arguments plus possibly the + :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` flag. + To get the actual number of positional arguments from *nargsf*, + use :c:func:`PyVectorcall_NARGS`. +- *kwnames* is a tuple containing the names of the keyword arguments; + in other words, the keys of the kwargs dict. + These names must be strings (instances of ``str`` or a subclass) + and they must be unique. + If there are no keyword arguments, then *kwnames* can instead be *NULL*. + +.. c:macro:: PY_VECTORCALL_ARGUMENTS_OFFSET + + If this flag is set in a vectorcall *nargsf* argument, the callee is allowed + to temporarily change ``args[-1]``. In other words, *args* points to + argument 1 (not 0) in the allocated vector. + The callee must restore the value of ``args[-1]`` before returning. + + For :c:func:`PyObject_VectorcallMethod`, this flag means instead that + ``args[0]`` may be changed. + + Whenever they can do so cheaply (without additional allocation), callers + are encouraged to use :const:`PY_VECTORCALL_ARGUMENTS_OFFSET`. + Doing so will allow callables such as bound methods to make their onward + calls (which include a prepended *self* argument) very efficiently. + +To call an object that implements vectorcall, use a :ref:`call API ` +function as with any other callable. +:c:func:`PyObject_Vectorcall` will usually be most efficient. + + +.. note:: + + In CPython 3.8, the vectorcall API and related functions were available + provisionally under names with a leading underscore: + ``_PyObject_Vectorcall``, ``_Py_TPFLAGS_HAVE_VECTORCALL``, + ``_PyObject_VectorcallMethod``, ``_PyVectorcall_Function``, + ``_PyObject_CallOneArg``, ``_PyObject_CallMethodNoArgs``, + ``_PyObject_CallMethodOneArg``. + Additionally, ``PyObject_VectorcallDict`` was available as + ``_PyObject_FastCallDict``. + The old names are still defined as aliases of the new, non-underscored names. + + +Recursion Control +................. + +When using *tp_call*, callees do not need to worry about +:ref:`recursion `: CPython uses +:c:func:`Py_EnterRecursiveCall` and :c:func:`Py_LeaveRecursiveCall` +for calls made using *tp_call*. + +For efficiency, this is not the case for calls done using vectorcall: +the callee should use *Py_EnterRecursiveCall* and *Py_LeaveRecursiveCall* +if needed. + + +Vectorcall Support API +...................... + +.. c:function:: Py_ssize_t PyVectorcall_NARGS(size_t nargsf) + + Given a vectorcall *nargsf* argument, return the actual number of + arguments. + Currently equivalent to:: + + (Py_ssize_t)(nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET) + + However, the function ``PyVectorcall_NARGS`` should be used to allow + for future extensions. + + This function is not part of the :ref:`limited API `. + + .. versionadded:: 3.8 + +.. c:function:: vectorcallfunc PyVectorcall_Function(PyObject *op) + + If *op* does not support the vectorcall protocol (either because the type + does not or because the specific instance does not), return *NULL*. + Otherwise, return the vectorcall function pointer stored in *op*. + This function never raises an exception. + + This is mostly useful to check whether or not *op* supports vectorcall, + which can be done by checking ``PyVectorcall_Function(op) != NULL``. + + This function is not part of the :ref:`limited API `. + + .. versionadded:: 3.8 + +.. c:function:: PyObject* PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict) + + Call *callable*'s :c:type:`vectorcallfunc` with positional and keyword + arguments given in a tuple and dict, respectively. + + This is a specialized function, intended to be put in the + :c:member:`~PyTypeObject.tp_call` slot or be used in an implementation of ``tp_call``. + It does not check the :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag + and it does not fall back to ``tp_call``. + + This function is not part of the :ref:`limited API `. + + .. versionadded:: 3.8 + + +.. _capi-call: + +Object Calling API +------------------ + +Various functions are available for calling a Python object. +Each converts its arguments to a convention supported by the called object – +either *tp_call* or vectorcall. +In order to do as litle conversion as possible, pick one that best fits +the format of data you have available. + +The following table summarizes the available functions; +please see individual documentation for details. + ++------------------------------------------+------------------+--------------------+---------------+ +| Function | callable | args | kwargs | ++==========================================+==================+====================+===============+ +| :c:func:`PyObject_Call` | ``PyObject *`` | tuple | dict/``NULL`` | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_CallNoArgs` | ``PyObject *`` | --- | --- | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_CallOneArg` | ``PyObject *`` | 1 object | --- | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_CallObject` | ``PyObject *`` | tuple/``NULL`` | --- | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_CallFunction` | ``PyObject *`` | format | --- | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_CallMethod` | obj + ``char*`` | format | --- | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_CallFunctionObjArgs` | ``PyObject *`` | variadic | --- | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_CallMethodObjArgs` | obj + name | variadic | --- | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_CallMethodNoArgs` | obj + name | --- | --- | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_CallMethodOneArg` | obj + name | 1 object | --- | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_Vectorcall` | ``PyObject *`` | vectorcall | vectorcall | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_VectorcallDict` | ``PyObject *`` | vectorcall | dict/``NULL`` | ++------------------------------------------+------------------+--------------------+---------------+ +| :c:func:`PyObject_VectorcallMethod` | arg + name | vectorcall | vectorcall | ++------------------------------------------+------------------+--------------------+---------------+ + + +.. c:function:: PyObject* PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs) + + Call a callable Python object *callable*, with arguments given by the + tuple *args*, and named arguments given by the dictionary *kwargs*. + + *args* must not be *NULL*; use an empty tuple if no arguments are needed. + If no named arguments are needed, *kwargs* can be *NULL*. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This is the equivalent of the Python expression: + ``callable(*args, **kwargs)``. + + +.. c:function:: PyObject* PyObject_CallNoArgs(PyObject *callable) + + Call a callable Python object *callable* without any arguments. It is the + most efficient way to call a callable Python object without any argument. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + .. versionadded:: 3.9 + + +.. c:function:: PyObject* PyObject_CallOneArg(PyObject *callable, PyObject *arg) + + Call a callable Python object *callable* with exactly 1 positional argument + *arg* and no keyword arguments. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This function is not part of the :ref:`limited API `. + + .. versionadded:: 3.9 + + +.. c:function:: PyObject* PyObject_CallObject(PyObject *callable, PyObject *args) + + Call a callable Python object *callable*, with arguments given by the + tuple *args*. If no arguments are needed, then *args* can be *NULL*. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This is the equivalent of the Python expression: ``callable(*args)``. + + +.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, const char *format, ...) + + Call a callable Python object *callable*, with a variable number of C arguments. + The C arguments are described using a :c:func:`Py_BuildValue` style format + string. The format can be *NULL*, indicating that no arguments are provided. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This is the equivalent of the Python expression: ``callable(*args)``. + + Note that if you only pass :c:type:`PyObject *` args, + :c:func:`PyObject_CallFunctionObjArgs` is a faster alternative. + + .. versionchanged:: 3.4 + The type of *format* was changed from ``char *``. + + +.. c:function:: PyObject* PyObject_CallMethod(PyObject *obj, const char *name, const char *format, ...) + + Call the method named *name* of object *obj* with a variable number of C + arguments. The C arguments are described by a :c:func:`Py_BuildValue` format + string that should produce a tuple. + + The format can be *NULL*, indicating that no arguments are provided. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This is the equivalent of the Python expression: + ``obj.name(arg1, arg2, ...)``. + + Note that if you only pass :c:type:`PyObject *` args, + :c:func:`PyObject_CallMethodObjArgs` is a faster alternative. + + .. versionchanged:: 3.4 + The types of *name* and *format* were changed from ``char *``. + + +.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ...) + + Call a callable Python object *callable*, with a variable number of + :c:type:`PyObject *` arguments. The arguments are provided as a variable number + of parameters followed by *NULL*. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This is the equivalent of the Python expression: + ``callable(arg1, arg2, ...)``. + + +.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) + + Call a method of the Python object *obj*, where the name of the method is given as a + Python string object in *name*. It is called with a variable number of + :c:type:`PyObject *` arguments. The arguments are provided as a variable number + of parameters followed by *NULL*. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + +.. c:function:: PyObject* PyObject_CallMethodNoArgs(PyObject *obj, PyObject *name) + + Call a method of the Python object *obj* without arguments, + where the name of the method is given as a Python string object in *name*. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This function is not part of the :ref:`limited API `. + + .. versionadded:: 3.9 + + +.. c:function:: PyObject* PyObject_CallMethodOneArg(PyObject *obj, PyObject *name, PyObject *arg) + + Call a method of the Python object *obj* with a single positional argument + *arg*, where the name of the method is given as a Python string object in + *name*. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This function is not part of the :ref:`limited API `. + + .. versionadded:: 3.9 + + +.. c:function:: PyObject* PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + Call a callable Python object *callable*. + The arguments are the same as for :c:type:`vectorcallfunc`. + If *callable* supports vectorcall_, this directly calls + the vectorcall function stored in *callable*. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This function is not part of the :ref:`limited API `. + + .. versionadded:: 3.9 + +.. c:function:: PyObject* PyObject_VectorcallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) + + Call *callable* with positional arguments passed exactly as in the vectorcall_ protocol, + but with keyword arguments passed as a dictionary *kwdict*. + The *args* array contains only the positional arguments. + + Regardless of which protocol is used internally, + a conversion of arguments needs to be done. + Therefore, this function should only be used if the caller + already has a dictionary ready to use for the keyword arguments, + but not a tuple for the positional arguments. + + This function is not part of the :ref:`limited API `. + + .. versionadded:: 3.9 + +.. c:function:: PyObject* PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + Call a method using the vectorcall calling convention. The name of the method + is given as a Python string *name*. The object whose method is called is + *args[0]*, and the *args* array starting at *args[1]* represents the arguments + of the call. There must be at least one positional argument. + *nargsf* is the number of positional arguments including *args[0]*, + plus :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` if the value of ``args[0]`` may + temporarily be changed. Keyword arguments can be passed just like in + :c:func:`PyObject_Vectorcall`. + + If the object has the :const:`Py_TPFLAGS_METHOD_DESCRIPTOR` feature, + this will call the unbound method object with the full + *args* vector as arguments. + + Return the result of the call on success, or raise an exception and return + *NULL* on failure. + + This function is not part of the :ref:`limited API `. + + .. versionadded:: 3.9 + + +Call Support API +---------------- + +.. c:function:: int PyCallable_Check(PyObject *o) + + Determine if the object *o* is callable. Return ``1`` if the object is callable + and ``0`` otherwise. This function always succeeds. diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 8eb6695e22de18..5eb313c89bfd59 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _capsules: @@ -15,7 +15,7 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:type:: PyCapsule This subtype of :c:type:`PyObject` represents an opaque value, useful for C - extension modules who need to pass an opaque value (as a :c:type:`void\*` + extension modules who need to pass an opaque value (as a :c:type:`void*` pointer) through Python code to other C code. It is often used to make a C function pointer defined in one module available to other modules, so the regular import mechanism can be used to access C APIs defined in dynamically @@ -40,15 +40,15 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: PyObject* PyCapsule_New(void *pointer, const char *name, PyCapsule_Destructor destructor) Create a :c:type:`PyCapsule` encapsulating the *pointer*. The *pointer* - argument may not be *NULL*. + argument may not be ``NULL``. - On failure, set an exception and return *NULL*. + On failure, set an exception and return ``NULL``. - The *name* string may either be *NULL* or a pointer to a valid C string. If - non-*NULL*, this string must outlive the capsule. (Though it is permitted to + The *name* string may either be ``NULL`` or a pointer to a valid C string. If + non-``NULL``, this string must outlive the capsule. (Though it is permitted to free it inside the *destructor*.) - If the *destructor* argument is not *NULL*, it will be called with the + If the *destructor* argument is not ``NULL``, it will be called with the capsule as its argument when it is destroyed. If this capsule will be stored as an attribute of a module, the *name* should @@ -59,20 +59,20 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: void* PyCapsule_GetPointer(PyObject *capsule, const char *name) Retrieve the *pointer* stored in the capsule. On failure, set an exception - and return *NULL*. + and return ``NULL``. The *name* parameter must compare exactly to the name stored in the capsule. - If the name stored in the capsule is *NULL*, the *name* passed in must also - be *NULL*. Python uses the C function :c:func:`strcmp` to compare capsule + If the name stored in the capsule is ``NULL``, the *name* passed in must also + be ``NULL``. Python uses the C function :c:func:`strcmp` to compare capsule names. .. c:function:: PyCapsule_Destructor PyCapsule_GetDestructor(PyObject *capsule) Return the current destructor stored in the capsule. On failure, set an - exception and return *NULL*. + exception and return ``NULL``. - It is legal for a capsule to have a *NULL* destructor. This makes a *NULL* + It is legal for a capsule to have a ``NULL`` destructor. This makes a ``NULL`` return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or :c:func:`PyErr_Occurred` to disambiguate. @@ -80,9 +80,9 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: void* PyCapsule_GetContext(PyObject *capsule) Return the current context stored in the capsule. On failure, set an - exception and return *NULL*. + exception and return ``NULL``. - It is legal for a capsule to have a *NULL* context. This makes a *NULL* + It is legal for a capsule to have a ``NULL`` context. This makes a ``NULL`` return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or :c:func:`PyErr_Occurred` to disambiguate. @@ -90,9 +90,9 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: const char* PyCapsule_GetName(PyObject *capsule) Return the current name stored in the capsule. On failure, set an exception - and return *NULL*. + and return ``NULL``. - It is legal for a capsule to have a *NULL* name. This makes a *NULL* return + It is legal for a capsule to have a ``NULL`` name. This makes a ``NULL`` return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or :c:func:`PyErr_Occurred` to disambiguate. @@ -107,13 +107,13 @@ Refer to :ref:`using-capsules` for more information on using these objects. import the module conventionally (using :c:func:`PyImport_ImportModule`). Return the capsule's internal *pointer* on success. On failure, set an - exception and return *NULL*. + exception and return ``NULL``. .. c:function:: int PyCapsule_IsValid(PyObject *capsule, const char *name) Determines whether or not *capsule* is a valid capsule. A valid capsule is - non-*NULL*, passes :c:func:`PyCapsule_CheckExact`, has a non-*NULL* pointer + non-``NULL``, passes :c:func:`PyCapsule_CheckExact`, has a non-``NULL`` pointer stored in it, and its internal name matches the *name* parameter. (See :c:func:`PyCapsule_GetPointer` for information on how capsule names are compared.) @@ -142,9 +142,9 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: int PyCapsule_SetName(PyObject *capsule, const char *name) - Set the name inside *capsule* to *name*. If non-*NULL*, the name must + Set the name inside *capsule* to *name*. If non-``NULL``, the name must outlive the capsule. If the previous *name* stored in the capsule was not - *NULL*, no attempt is made to free it. + ``NULL``, no attempt is made to free it. Return ``0`` on success. Return nonzero and set an exception on failure. @@ -152,6 +152,6 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: int PyCapsule_SetPointer(PyObject *capsule, void *pointer) Set the void pointer inside *capsule* to *pointer*. The pointer may not be - *NULL*. + ``NULL``. Return ``0`` on success. Return nonzero and set an exception on failure. diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 427259cc24d8d8..8408f7e398db7b 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _cell-objects: @@ -27,13 +27,13 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: int PyCell_Check(ob) - Return true if *ob* is a cell object; *ob* must not be *NULL*. + Return true if *ob* is a cell object; *ob* must not be ``NULL``. .. c:function:: PyObject* PyCell_New(PyObject *ob) Create and return a new cell object containing the value *ob*. The parameter may - be *NULL*. + be ``NULL``. .. c:function:: PyObject* PyCell_Get(PyObject *cell) @@ -44,19 +44,19 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: PyObject* PyCell_GET(PyObject *cell) Return the contents of the cell *cell*, but without checking that *cell* is - non-*NULL* and a cell object. + non-``NULL`` and a cell object. .. c:function:: int PyCell_Set(PyObject *cell, PyObject *value) Set the contents of the cell object *cell* to *value*. This releases the - reference to any current content of the cell. *value* may be *NULL*. *cell* - must be non-*NULL*; if it is not a cell object, ``-1`` will be returned. On + reference to any current content of the cell. *value* may be ``NULL``. *cell* + must be non-``NULL``; if it is not a cell object, ``-1`` will be returned. On success, ``0`` will be returned. .. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) Sets the value of the cell object *cell* to *value*. No reference counts are - adjusted, and no checks are made for safety; *cell* must be non-*NULL* and must + adjusted, and no checks are made for safety; *cell* must be non-``NULL`` and must be a cell object. diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 10d89f297c843f..6f8c41ccbf6e85 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _codeobjects: @@ -35,11 +35,16 @@ bound into a function. .. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) - Return a new code object. If you need a dummy code object to - create a frame, use :c:func:`PyCode_NewEmpty` instead. Calling - :c:func:`PyCode_New` directly can bind you to a precise Python - version since the definition of the bytecode changes often. + Return a new code object. If you need a dummy code object to create a frame, + use :c:func:`PyCode_NewEmpty` instead. Calling :c:func:`PyCode_New` directly + can bind you to a precise Python version since the definition of the bytecode + changes often. +.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) + + Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positional-only arguments. + + .. versionadded:: 3.8 .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index c55f19970e125d..172dcb326a4bc4 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -21,7 +21,7 @@ Codec registry and support functions *object* is passed through the encoder function found for the given *encoding* using the error handling method defined by *errors*. *errors* may - be *NULL* to use the default method defined for the codec. Raises a + be ``NULL`` to use the default method defined for the codec. Raises a :exc:`LookupError` if no encoder can be found. .. c:function:: PyObject* PyCodec_Decode(PyObject *object, const char *encoding, const char *errors) @@ -30,7 +30,7 @@ Codec registry and support functions *object* is passed through the decoder function found for the given *encoding* using the error handling method defined by *errors*. *errors* may - be *NULL* to use the default method defined for the codec. Raises a + be ``NULL`` to use the default method defined for the codec. Raises a :exc:`LookupError` if no encoder can be found. @@ -40,7 +40,7 @@ Codec lookup API In the following functions, the *encoding* string is looked up converted to all lower-case characters, which makes encodings looked up through this mechanism effectively case-insensitive. If no codec is found, a :exc:`KeyError` is set -and *NULL* returned. +and ``NULL`` returned. .. c:function:: PyObject* PyCodec_Encoder(const char *encoding) @@ -92,7 +92,7 @@ Registry API for Unicode encoding error handlers .. c:function:: PyObject* PyCodec_LookupError(const char *name) Lookup the error handling callback function registered under *name*. As a - special case *NULL* can be passed, in which case the error handling callback + special case ``NULL`` can be passed, in which case the error handling callback for "strict" will be returned. .. c:function:: PyObject* PyCodec_StrictErrors(PyObject *exc) diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index fc63b57a85521c..06dbb2572725ee 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _complexobjects: @@ -129,4 +129,10 @@ Complex Numbers as Python Objects If *op* is not a Python complex number object but has a :meth:`__complex__` method, this method will first be called to convert *op* to a Python complex - number object. Upon failure, this method returns ``-1.0`` as a real value. + number object. If ``__complex__()`` is not defined then it falls back to + :meth:`__float__`. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. Upon failure, this method returns ``-1.0`` as a real + value. + + .. versionchanged:: 3.8 + Use :meth:`__index__` if available. diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 9558a4a583cba5..bf263d6e4c2641 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _concrete: @@ -17,8 +17,8 @@ dictionary, use :c:func:`PyDict_Check`. The chapter is structured like the .. warning:: While the functions described in this chapter carefully check the type of the - objects which are passed in, many of them do not check for *NULL* being passed - instead of a valid object. Allowing *NULL* to be passed in can cause memory + objects which are passed in, many of them do not check for ``NULL`` being passed + instead of a valid object. Allowing ``NULL`` to be passed in can cause memory access violations and immediate termination of the interpreter. @@ -115,3 +115,4 @@ Other Objects coro.rst contextvars.rst datetime.rst + decimal.rst diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index c344c8d71ae32d..9c088814314a81 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _contextvarsobjects: @@ -60,17 +60,17 @@ Type-check macros: .. c:function:: int PyContext_CheckExact(PyObject *o) Return true if *o* is of type :c:data:`PyContext_Type`. *o* must not be - *NULL*. This function always succeeds. + ``NULL``. This function always succeeds. .. c:function:: int PyContextVar_CheckExact(PyObject *o) Return true if *o* is of type :c:data:`PyContextVar_Type`. *o* must not be - *NULL*. This function always succeeds. + ``NULL``. This function always succeeds. .. c:function:: int PyContextToken_CheckExact(PyObject *o) Return true if *o* is of type :c:data:`PyContextToken_Type`. - *o* must not be *NULL*. This function always succeeds. + *o* must not be ``NULL``. This function always succeeds. Context object management functions: @@ -101,11 +101,6 @@ Context object management functions: current context for the current thread. Returns ``0`` on success, and ``-1`` on error. -.. c:function:: int PyContext_ClearFreeList() - - Clear the context variable free list. Return the total number of - freed items. This function always succeeds. - Context variable functions: diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index c46722d782a2fb..efbaa52e2dc466 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _string-conversion: @@ -27,12 +27,8 @@ not. The wrappers ensure that *str*[*size*-1] is always ``'\0'`` upon return. They never write more than *size* bytes (including the trailing ``'\0'``) into str. -Both functions require that ``str != NULL``, ``size > 0`` and ``format != -NULL``. - -If the platform doesn't have :c:func:`vsnprintf` and the buffer size needed to -avoid truncation exceeds *size* by more than 512 bytes, Python aborts with a -*Py_FatalError*. +Both functions require that ``str != NULL``, ``size > 0``, ``format != NULL`` +and ``size < INT_MAX``. The return value (*rv*) for these functions should be interpreted as follows: @@ -48,8 +44,8 @@ The return value (*rv*) for these functions should be interpreted as follows: this case too, but the rest of *str* is undefined. The exact cause of the error depends on the underlying platform. -The following functions provide locale-independent string to number conversions. +The following functions provide locale-independent string to number conversions. .. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) @@ -95,25 +91,25 @@ The following functions provide locale-independent string to number conversions. must be 0 and is ignored. The ``'r'`` format code specifies the standard :func:`repr` format. - *flags* can be zero or more of the values *Py_DTSF_SIGN*, - *Py_DTSF_ADD_DOT_0*, or *Py_DTSF_ALT*, or-ed together: + *flags* can be zero or more of the values ``Py_DTSF_SIGN``, + ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together: - * *Py_DTSF_SIGN* means to always precede the returned string with a sign + * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign character, even if *val* is non-negative. - * *Py_DTSF_ADD_DOT_0* means to ensure that the returned string will not look + * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look like an integer. - * *Py_DTSF_ALT* means to apply "alternate" formatting rules. See the + * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules. See the documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for details. - If *ptype* is non-NULL, then the value it points to will be set to one of - *Py_DTST_FINITE*, *Py_DTST_INFINITE*, or *Py_DTST_NAN*, signifying that + If *ptype* is non-``NULL``, then the value it points to will be set to one of + ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that *val* is a finite number, an infinite number, or not a number, respectively. The return value is a pointer to *buffer* with the converted string or - *NULL* if the conversion failed. The caller is responsible for freeing the + ``NULL`` if the conversion failed. The caller is responsible for freeing the returned string by calling :c:func:`PyMem_Free`. .. versionadded:: 3.1 diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst index 2fe50b5d8c4473..2260944a9a93a9 100644 --- a/Doc/c-api/coro.rst +++ b/Doc/c-api/coro.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _coro-objects: @@ -23,7 +23,7 @@ return. .. c:function:: int PyCoro_CheckExact(PyObject *ob) - Return true if *ob*'s type is *PyCoro_Type*; *ob* must not be *NULL*. + Return true if *ob*'s type is :c:type:`PyCoro_Type`; *ob* must not be ``NULL``. .. c:function:: PyObject* PyCoro_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) @@ -31,4 +31,4 @@ return. Create and return a new coroutine object based on the *frame* object, with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument - must not be *NULL*. + must not be ``NULL``. diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 78724619ea3c52..bd4f1ff446bcf4 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _datetimeobjects: @@ -28,61 +28,61 @@ Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be *NULL*. + :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. .. c:function:: int PyDate_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType`. *ob* must not be - *NULL*. + ``NULL``. .. c:function:: int PyDateTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be *NULL*. + :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. .. c:function:: int PyDateTime_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType`. *ob* must not - be *NULL*. + be ``NULL``. .. c:function:: int PyTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be *NULL*. + :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. .. c:function:: int PyTime_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType`. *ob* must not be - *NULL*. + ``NULL``. .. c:function:: int PyDelta_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be *NULL*. + :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. .. c:function:: int PyDelta_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType`. *ob* must not be - *NULL*. + ``NULL``. .. c:function:: int PyTZInfo_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be *NULL*. + :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. .. c:function:: int PyTZInfo_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType`. *ob* must not be - *NULL*. + ``NULL``. Macros to create objects: @@ -98,12 +98,28 @@ Macros to create objects: minute, second and microsecond. +.. c:function:: PyObject* PyDateTime_FromDateAndTimeAndFold(int year, int month, int day, int hour, int minute, int second, int usecond, int fold) + + Return a :class:`datetime.datetime` object with the specified year, month, day, hour, + minute, second, microsecond and fold. + + .. versionadded:: 3.6 + + .. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond) Return a :class:`datetime.time` object with the specified hour, minute, second and microsecond. +.. c:function:: PyObject* PyTime_FromTimeAndFold(int hour, int minute, int second, int usecond, int fold) + + Return a :class:`datetime.time` object with the specified hour, minute, second, + microsecond and fold. + + .. versionadded:: 3.6 + + .. c:function:: PyObject* PyDelta_FromDSU(int days, int seconds, int useconds) Return a :class:`datetime.timedelta` object representing the given number @@ -128,7 +144,7 @@ Macros to create objects: Macros to extract fields from date objects. The argument must be an instance of :c:data:`PyDateTime_Date`, including subclasses (such as -:c:data:`PyDateTime_DateTime`). The argument must not be *NULL*, and the type is +:c:data:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) @@ -148,7 +164,7 @@ not checked: Macros to extract fields from datetime objects. The argument must be an instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument -must not be *NULL*, and the type is not checked: +must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) @@ -171,7 +187,7 @@ must not be *NULL*, and the type is not checked: Macros to extract fields from time objects. The argument must be an instance of -:c:data:`PyDateTime_Time`, including subclasses. The argument must not be *NULL*, +:c:data:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) @@ -196,7 +212,7 @@ and the type is not checked: Macros to extract fields from time delta objects. The argument must be an instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must -not be *NULL*, and the type is not checked: +not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o) diff --git a/Doc/c-api/decimal.rst b/Doc/c-api/decimal.rst new file mode 100644 index 00000000000000..f530571ebae577 --- /dev/null +++ b/Doc/c-api/decimal.rst @@ -0,0 +1,231 @@ +.. sectionauthor:: Stefan Krah + +.. highlight:: c + + +Decimal capsule API +=================== + +Capsule API functions can be used in the same manner as regular library +functions, provided that the API has been initialized. + + +Initialize +---------- + +Typically, a C extension module that uses the decimal API will do these +steps in its init function: + +.. code-block:: + + #include "pydecimal.h" + + static int decimal_initialized = 0; + if (!decimal_initialized) { + if (import_decimal() < 0) { + return NULL; + } + + decimal_initialized = 1; + } + + +Type checking, predicates, accessors +------------------------------------ + +.. c:function:: int PyDec_TypeCheck(const PyObject *dec) + + Return 1 if ``dec`` is a Decimal, 0 otherwise. This function does not set + any exceptions. + + +.. c:function:: int PyDec_IsSpecial(const PyObject *dec) + + Return 1 if ``dec`` is ``NaN``, ``sNaN`` or ``Infinity``, 0 otherwise. + + Set TypeError and return -1 if ``dec`` is not a Decimal. It is guaranteed that + this is the only failure mode, so if ``dec`` has already been type-checked, no + errors can occur and the function can be treated as a simple predicate. + + +.. c:function:: int PyDec_IsNaN(const PyObject *dec) + + Return 1 if ``dec`` is ``NaN`` or ``sNaN``, 0 otherwise. + + Set TypeError and return -1 if ``dec`` is not a Decimal. It is guaranteed that + this is the only failure mode, so if ``dec`` has already been type-checked, no + errors can occur and the function can be treated as a simple predicate. + + +.. c:function:: int PyDec_IsInfinite(const PyObject *dec) + + Return 1 if ``dec`` is ``Infinity``, 0 otherwise. + + Set TypeError and return -1 if ``dec`` is not a Decimal. It is guaranteed that + this is the only failure mode, so if ``dec`` has already been type-checked, no + errors can occur and the function can be treated as a simple predicate. + + +.. c:function:: int64_t PyDec_GetDigits(const PyObject *dec) + + Return the number of digits in the coefficient. For ``Infinity``, the + number of digits is always zero. Typically, the same applies to ``NaN`` + and ``sNaN``, but both of these can have a payload that is equivalent to + a coefficient. Therefore, ``NaNs`` can have a nonzero return value. + + Set TypeError and return -1 if ``dec`` is not a Decimal. It is guaranteed that + this is the only failure mode, so if ``dec`` has already been type-checked, no + errors can occur and the function can be treated as a simple accessor. + + +Exact conversions between decimals and primitive C types +-------------------------------------------------------- + +This API supports conversions for decimals with a coefficient up to 38 digits. + +Data structures +~~~~~~~~~~~~~~~ + +The conversion functions use the following status codes and data structures: + +.. code-block:: + + /* status cases for getting a triple */ + enum mpd_triple_class { + MPD_TRIPLE_NORMAL, + MPD_TRIPLE_INF, + MPD_TRIPLE_QNAN, + MPD_TRIPLE_SNAN, + MPD_TRIPLE_ERROR, + }; + + typedef struct { + enum mpd_triple_class tag; + uint8_t sign; + uint64_t hi; + uint64_t lo; + int64_t exp; + } mpd_uint128_triple_t; + +The status cases are explained below. ``sign`` is 0 for positive and 1 for negative. +``((uint128_t)hi << 64) + lo`` is the coefficient, ``exp`` is the exponent. + +The data structure is called "triple" because the decimal triple (sign, coeff, exp) +is an established term and (``hi``, ``lo``) represents a single ``uint128_t`` coefficient. + + +Functions +~~~~~~~~~ + +.. c:function:: mpd_uint128_triple_t PyDec_AsUint128Triple(const PyObject *dec) + + Convert a decimal to a triple. As above, it is guaranteed that the only + Python failure mode is a TypeError, checks can be omitted if the type is + known. + + For simplicity, the usage of the function and all special cases are + explained in code form and comments: + +.. code-block:: + + triple = PyDec_AsUint128Triple(dec); + switch (triple.tag) { + case MPD_TRIPLE_QNAN: + /* + * Success: handle a quiet NaN. + * 1) triple.sign is 0 or 1. + * 2) triple.exp is always 0. + * 3) If triple.hi or triple.lo are nonzero, the NaN has a payload. + */ + break; + + case MPD_TRIPLE_SNAN: + /* + * Success: handle a signaling NaN. + * 1) triple.sign is 0 or 1. + * 2) triple.exp is always 0. + * 3) If triple.hi or triple.lo are nonzero, the sNaN has a payload. + */ + break; + + case MPD_TRIPLE_INF: + /* + * Success: handle Infinity. + * 1) triple.sign is 0 or 1. + * 2) triple.exp is always 0. + * 3) triple.hi and triple.lo are always zero. + */ + break; + + case MPD_TRIPLE_NORMAL: + /* Success: handle a finite value. */ + break; + + case MPD_TRIPLE_ERROR: + /* TypeError check: can be omitted if the type of dec is known. */ + if (PyErr_Occurred()) { + return NULL; + } + + /* Too large for conversion. PyDec_AsUint128Triple() does not set an + exception so applications can choose themselves. Typically this + would be a ValueError. */ + PyErr_SetString(PyExc_ValueError, + "value out of bounds for a uint128 triple"); + return NULL; + } + +.. c:function:: PyObject *PyDec_FromUint128Triple(const mpd_uint128_triple_t *triple) + + Create a decimal from a triple. The following rules must be observed for + initializing the triple: + + 1) ``triple.sign`` must always be 0 (for positive) or 1 (for negative). + + 2) ``MPD_TRIPLE_QNAN``: ``triple.exp`` must be 0. If ``triple.hi`` or ``triple.lo`` + are nonzero, create a ``NaN`` with a payload. + + 3) ``MPD_TRIPLE_SNAN``: ``triple.exp`` must be 0. If ``triple.hi`` or ``triple.lo`` + are nonzero, create an ``sNaN`` with a payload. + + 4) ``MPD_TRIPLE_INF``: ``triple.exp``, ``triple.hi`` and ``triple.lo`` must be zero. + + 5) ``MPD_TRIPLE_NORMAL``: ``MPD_MIN_ETINY + 38 < triple.exp < MPD_MAX_EMAX - 38``. + ``triple.hi`` and ``triple.lo`` can be chosen freely. + + 6) ``MPD_TRIPLE_ERROR``: It is always an error to set this tag. + + + If one of the above conditions is not met, the function returns ``NaN`` if + the ``InvalidOperation`` trap is not set in the thread local context. Otherwise, + it sets the ``InvalidOperation`` exception and returns NULL. + + Additionally, though extremely unlikely give the small allocation sizes, + the function can set ``MemoryError`` and return ``NULL``. + + +Advanced API +------------ + +This API enables the use of ``libmpdec`` functions. Since Python is compiled with +hidden symbols, the API requires an external libmpdec and the ``mpdecimal.h`` +header. + + +Functions +~~~~~~~~~ + +.. c:function:: PyObject *PyDec_Alloc(void) + + Return a new decimal that can be used in the ``result`` position of ``libmpdec`` + functions. + +.. c:function:: mpd_t *PyDec_Get(PyObject *v) + + Get a pointer to the internal ``mpd_t`` of the decimal. Decimals are immutable, + so this function must only be used on a new Decimal that has been created by + PyDec_Alloc(). + +.. c:function:: const mpd_t *PyDec_GetConst(const PyObject *v) + + Get a pointer to the constant internal ``mpd_t`` of the decimal. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index c8f6fa5bcd7fc8..1005140c7acb3a 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _descriptor-objects: diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 0ced5a5fd00170..769484134ed515 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _dictobjects: @@ -33,7 +33,7 @@ Dictionary Objects .. c:function:: PyObject* PyDict_New() - Return a new empty dictionary, or *NULL* on failure. + Return a new empty dictionary, or ``NULL`` on failure. .. c:function:: PyObject* PyDictProxy_New(PyObject *mapping) @@ -62,19 +62,20 @@ Dictionary Objects .. c:function:: int PyDict_SetItem(PyObject *p, PyObject *key, PyObject *val) - Insert *value* into the dictionary *p* with a key of *key*. *key* must be + Insert *val* into the dictionary *p* with a key of *key*. *key* must be :term:`hashable`; if it isn't, :exc:`TypeError` will be raised. Return - ``0`` on success or ``-1`` on failure. + ``0`` on success or ``-1`` on failure. This function *does not* steal a + reference to *val*. .. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) .. index:: single: PyUnicode_FromString() - Insert *value* into the dictionary *p* using *key* as a key. *key* should - be a :c:type:`const char\*`. The key object is created using + Insert *val* into the dictionary *p* using *key* as a key. *key* should + be a :c:type:`const char*`. The key object is created using ``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on - failure. + failure. This function *does not* steal a reference to *val*. .. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) @@ -92,26 +93,30 @@ Dictionary Objects .. c:function:: PyObject* PyDict_GetItem(PyObject *p, PyObject *key) - Return the object from dictionary *p* which has a key *key*. Return *NULL* + Return the object from dictionary *p* which has a key *key*. Return ``NULL`` if the key *key* is not present, but *without* setting an exception. Note that exceptions which occur while calling :meth:`__hash__` and :meth:`__eq__` methods will get suppressed. To get error reporting use :c:func:`PyDict_GetItemWithError()` instead. + .. versionchanged:: 3.10 + Calling this API without :term:`GIL` held had been allowed for historical + reason. It is no longer allowed. + .. c:function:: PyObject* PyDict_GetItemWithError(PyObject *p, PyObject *key) Variant of :c:func:`PyDict_GetItem` that does not suppress - exceptions. Return *NULL* **with** an exception set if an exception - occurred. Return *NULL* **without** an exception set if the key + exceptions. Return ``NULL`` **with** an exception set if an exception + occurred. Return ``NULL`` **without** an exception set if the key wasn't present. .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a - :c:type:`const char\*`, rather than a :c:type:`PyObject\*`. + :c:type:`const char*`, rather than a :c:type:`PyObject*`. Note that exceptions which occur while calling :meth:`__hash__` and :meth:`__eq__` methods and creating a temporary string object @@ -160,8 +165,8 @@ Dictionary Objects prior to the first call to this function to start the iteration; the function returns true for each pair in the dictionary, and false once all pairs have been reported. The parameters *pkey* and *pvalue* should either - point to :c:type:`PyObject\*` variables that will be filled in with each key - and value, respectively, or may be *NULL*. Any references returned through + point to :c:type:`PyObject*` variables that will be filled in with each key + and value, respectively, or may be ``NULL``. Any references returned through them are borrowed. *ppos* should not be altered during iteration. Its value represents offsets within the internal dictionary structure, and since the structure is sparse, the offsets are not consecutive. @@ -231,10 +236,3 @@ Dictionary Objects for key, value in seq2: if override or key not in a: a[key] = value - - -.. c:function:: int PyDict_ClearFreeList() - - Clear the free list. Return the total number of freed items. - - .. versionadded:: 3.3 diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8c5f66cbef722b..247b6d68eceae9 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _exceptionhandling: @@ -13,15 +13,15 @@ exception handling. It works somewhat like the POSIX :c:data:`errno` variable: there is a global indicator (per thread) of the last error that occurred. Most C API functions don't clear this on success, but will set it to indicate the cause of the error on failure. Most C API functions also return an error -indicator, usually *NULL* if they are supposed to return a pointer, or ``-1`` +indicator, usually ``NULL`` if they are supposed to return a pointer, or ``-1`` if they return an integer (exception: the :c:func:`PyArg_\*` functions return ``1`` for success and ``0`` for failure). Concretely, the error indicator consists of three object pointers: the exception's type, the exception's value, and the traceback object. Any -of those pointers can be NULL if non-set (although some combinations are -forbidden, for example you can't have a non-NULL traceback if the exception -type is NULL). +of those pointers can be ``NULL`` if non-set (although some combinations are +forbidden, for example you can't have a non-``NULL`` traceback if the exception +type is ``NULL``). When a function must fail because some function it called failed, it generally doesn't set the error indicator; the function it called already set it. It is @@ -53,8 +53,8 @@ Printing and clearing .. c:function:: void PyErr_PrintEx(int set_sys_last_vars) Print a standard traceback to ``sys.stderr`` and clear the error indicator. - **Unless** the error is a ``SystemExit``. In that case the no traceback - is printed and Python process will exit with the error code specified by + **Unless** the error is a ``SystemExit``, in that case no traceback is + printed and the Python process will exit with the error code specified by the ``SystemExit`` instance. Call this function **only** when the error indicator is set. Otherwise it @@ -72,6 +72,9 @@ Printing and clearing .. c:function:: void PyErr_WriteUnraisable(PyObject *obj) + Call :func:`sys.unraisablehook` using the current exception and *obj* + argument. + This utility function prints a warning message to ``sys.stderr`` when an exception has been set but it is impossible for the interpreter to actually raise the exception. It is used, for example, when an exception occurs in an @@ -81,13 +84,15 @@ Printing and clearing in which the unraisable exception occurred. If possible, the repr of *obj* will be printed in the warning message. + An exception must be set when calling this function. + Raising exceptions ================== These functions help you set the current thread's error indicator. For convenience, some of these functions will always return a -NULL pointer for use in a ``return`` statement. +``NULL`` pointer for use in a ``return`` statement. .. c:function:: void PyErr_SetString(PyObject *type, const char *message) @@ -106,7 +111,7 @@ NULL pointer for use in a ``return`` statement. .. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) - This function sets the error indicator and returns *NULL*. *exception* + This function sets the error indicator and returns ``NULL``. *exception* should be a Python exception class. The *format* and subsequent parameters help format the error message; they have the same meaning and values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded @@ -135,7 +140,7 @@ NULL pointer for use in a ``return`` statement. .. c:function:: PyObject* PyErr_NoMemory() - This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns *NULL* + This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns ``NULL`` so an object allocation function can write ``return PyErr_NoMemory();`` when it runs out of memory. @@ -151,7 +156,7 @@ NULL pointer for use in a ``return`` statement. and then calls ``PyErr_SetObject(type, object)``. On Unix, when the :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, - leaves it set to that. The function always returns *NULL*, so a wrapper + leaves it set to that. The function always returns ``NULL``, so a wrapper function around a system call can write ``return PyErr_SetFromErrno(type);`` when the system call returns an error. @@ -159,7 +164,7 @@ NULL pointer for use in a ``return`` statement. .. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if - *filenameObject* is not *NULL*, it is passed to the constructor of *type* as + *filenameObject* is not ``NULL``, it is passed to the constructor of *type* as a third parameter. In the case of :exc:`OSError` exception, this is used to define the :attr:`filename` attribute of the exception instance. @@ -190,7 +195,7 @@ NULL pointer for use in a ``return`` statement. then it constructs a tuple object whose first item is the *ierr* value and whose second item is the corresponding error message (gotten from :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, - object)``. This function always returns *NULL*. + object)``. This function always returns ``NULL``. .. availability:: Windows. @@ -299,7 +304,7 @@ an error value). .. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) Issue a warning message. The *category* argument is a warning category (see - below) or *NULL*; the *message* argument is a UTF-8 encoded string. *stack_level* is a + below) or ``NULL``; the *message* argument is a UTF-8 encoded string. *stack_level* is a positive number giving a number of stack frames; the warning will be issued from the currently executing line of code in that stack frame. A *stack_level* of 1 is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, @@ -315,7 +320,7 @@ an error value). :mod:`warnings` module and the :option:`-W` option in the command line documentation. There is no C API for warning control. -.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *msg, PyObject *name, PyObject *path) +.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) Much like :c:func:`PyErr_SetImportError` but this function allows for specifying a subclass of :exc:`ImportError` to raise. @@ -328,7 +333,7 @@ an error value). Issue a warning message with explicit control over all warning attributes. This is a straightforward wrapper around the Python function :func:`warnings.warn_explicit`, see there for more information. The *module* - and *registry* arguments may be set to *NULL* to get the default effect + and *registry* arguments may be set to ``NULL`` to get the default effect described there. .. versionadded:: 3.4 @@ -353,7 +358,7 @@ an error value). .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and pass *source* to :func:`warnings.WarningMessage`. + :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. .. versionadded:: 3.6 @@ -365,10 +370,12 @@ Querying the error indicator Test whether the error indicator is set. If set, return the exception *type* (the first argument to the last call to one of the :c:func:`PyErr_Set\*` - functions or to :c:func:`PyErr_Restore`). If not set, return *NULL*. You do not + functions or to :c:func:`PyErr_Restore`). If not set, return ``NULL``. You do not own a reference to the return value, so you do not need to :c:func:`Py_DECREF` it. + The caller must hold the GIL. + .. note:: Do not compare the return value to a specific exception; use @@ -395,9 +402,9 @@ Querying the error indicator .. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) Retrieve the error indicator into three variables whose addresses are passed. - If the error indicator is not set, set all three variables to *NULL*. If it is + If the error indicator is not set, set all three variables to ``NULL``. If it is set, it will be cleared and you own a reference to each object retrieved. The - value and traceback object may be *NULL* even when the type object is not. + value and traceback object may be ``NULL`` even when the type object is not. .. note:: @@ -417,8 +424,8 @@ Querying the error indicator .. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are *NULL*, the error - indicator is cleared. Do not pass a *NULL* type and non-*NULL* value or + already set, it is cleared first. If the objects are ``NULL``, the error + indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or traceback. The exception type should be a class. Do not pass an invalid exception type or value. (Violating these rules will cause subtle problems later.) This call takes away a reference to each object: you must own a @@ -457,7 +464,7 @@ Querying the error indicator Retrieve the exception info, as known from ``sys.exc_info()``. This refers to an exception that was *already caught*, not to an exception that was freshly raised. Returns new references for the three objects, any of which - may be *NULL*. Does not modify the exception info state. + may be ``NULL``. Does not modify the exception info state. .. note:: @@ -474,7 +481,7 @@ Querying the error indicator Set the exception info, as known from ``sys.exc_info()``. This refers to an exception that was *already caught*, not to an exception that was freshly raised. This function steals the references of the arguments. - To clear the exception state, pass *NULL* for all three arguments. + To clear the exception state, pass ``NULL`` for all three arguments. For general rules about the three arguments, see :c:func:`PyErr_Restore`. .. note:: @@ -514,13 +521,13 @@ Signal Handling single: SIGINT single: KeyboardInterrupt (built-in exception) - This function simulates the effect of a :const:`SIGINT` signal arriving --- the - next time :c:func:`PyErr_CheckSignals` is called, :exc:`KeyboardInterrupt` will - be raised. It may be called without holding the interpreter lock. - - .. % XXX This was described as obsolete, but is used in - .. % _thread.interrupt_main() (used from IDLE), so it's still needed. + Simulate the effect of a :const:`SIGINT` signal arriving. The next time + :c:func:`PyErr_CheckSignals` is called, the Python signal handler for + :const:`SIGINT` will be called. + If :const:`SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. .. c:function:: int PySignal_SetWakeupFd(int fd) @@ -544,7 +551,7 @@ Exception Classes This utility function creates and returns a new exception class. The *name* argument must be the name of the new exception, a C string of the form - ``module.classname``. The *base* and *dict* arguments are normally *NULL*. + ``module.classname``. The *base* and *dict* arguments are normally ``NULL``. This creates a class object derived from :exc:`Exception` (accessible in C as :c:data:`PyExc_Exception`). @@ -558,7 +565,7 @@ Exception Classes .. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) Same as :c:func:`PyErr_NewException`, except that the new exception class can - easily be given a docstring: If *doc* is non-*NULL*, it will be used as the + easily be given a docstring: If *doc* is non-``NULL``, it will be used as the docstring for the exception class. .. versionadded:: 3.2 @@ -571,7 +578,7 @@ Exception Objects Return the traceback associated with the exception as a new reference, as accessible from Python through :attr:`__traceback__`. If there is no - traceback associated, this returns *NULL*. + traceback associated, this returns ``NULL``. .. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) @@ -585,12 +592,12 @@ Exception Objects Return the context (another exception instance during whose handling *ex* was raised) associated with the exception as a new reference, as accessible from Python through :attr:`__context__`. If there is no context associated, this - returns *NULL*. + returns ``NULL``. .. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) - Set the context associated with the exception to *ctx*. Use *NULL* to clear + Set the context associated with the exception to *ctx*. Use ``NULL`` to clear it. There is no type check to make sure that *ctx* is an exception instance. This steals a reference to *ctx*. @@ -604,7 +611,7 @@ Exception Objects .. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) - Set the cause associated with the exception to *cause*. Use *NULL* to clear + Set the cause associated with the exception to *cause*. Use ``NULL`` to clear it. There is no type check to make sure that *cause* is either an exception instance or :const:`None`. This steals a reference to *cause*. @@ -630,11 +637,21 @@ The following functions are used to create and modify Unicode exceptions from C. *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are UTF-8 encoded strings. + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", ...)``. + .. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) Create a :class:`UnicodeTranslateError` object with the attributes *object*, *length*, *start*, *end* and *reason*. *reason* is a UTF-8 encoded string. + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeTranslateError, "Onns", ...)``. + .. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) @@ -651,7 +668,7 @@ The following functions are used to create and modify Unicode exceptions from C. int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) Get the *start* attribute of the given exception object and place it into - *\*start*. *start* must not be *NULL*. Return ``0`` on success, ``-1`` on + *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. .. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) @@ -666,7 +683,7 @@ The following functions are used to create and modify Unicode exceptions from C. int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) Get the *end* attribute of the given exception object and place it into - *\*end*. *end* must not be *NULL*. Return ``0`` on success, ``-1`` on + *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. .. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) @@ -690,6 +707,8 @@ The following functions are used to create and modify Unicode exceptions from C. ``0`` on success, ``-1`` on failure. +.. _recursion: + Recursion Control ================= @@ -697,6 +716,8 @@ These two functions provide a way to perform safe recursive calls at the C level, both in the core and in extension modules. They are needed if the recursive code does not necessarily invoke Python code (which tracks its recursion depth automatically). +They are also not needed for *tp_call* implementations +because the :ref:`call protocol ` takes care of recursion handling. .. c:function:: int Py_EnterRecursiveCall(const char *where) @@ -710,15 +731,21 @@ recursion depth automatically). case, a :exc:`RecursionError` is set and a nonzero value is returned. Otherwise, zero is returned. - *where* should be a string such as ``" in instance check"`` to be - concatenated to the :exc:`RecursionError` message caused by the recursion + *where* should be a UTF-8 encoded string such as ``" in instance check"`` to + be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. -.. c:function:: void Py_LeaveRecursiveCall() + .. versionchanged:: 3.9 + This function is now also available in the limited API. + +.. c:function:: void Py_LeaveRecursiveCall(void) Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each *successful* invocation of :c:func:`Py_EnterRecursiveCall`. + .. versionchanged:: 3.9 + This function is now also available in the limited API. + Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires special recursion handling. In addition to protecting the stack, :c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The @@ -756,7 +783,7 @@ Standard Exceptions All standard Python exceptions are available as global variables whose names are ``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +:c:type:`PyObject*`; they are all class objects. For completeness, here are all the variables: .. index:: @@ -804,7 +831,6 @@ the variables: single: PyExc_SystemError single: PyExc_SystemExit single: PyExc_TabError - single: PyExc_TargetScopeError single: PyExc_TimeoutError single: PyExc_TypeError single: PyExc_UnboundLocalError @@ -906,8 +932,6 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TabError` | :exc:`TabError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TargetScopeError` | :exc:`TargetScopeError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | @@ -969,9 +993,6 @@ Notes: This is a base class for other standard exceptions. (2) - This is the same as :exc:`weakref.ReferenceError`. - -(3) Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. @@ -982,7 +1003,7 @@ Standard Warning Categories All standard Python warning categories are available as global variables whose names are ``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +:c:type:`PyObject*`; they are all class objects. For completeness, here are all the variables: .. index:: diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index 6f2ecee51fd86a..ea027ee975c651 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _fileobjects: @@ -8,7 +8,7 @@ File Objects .. index:: object: file These APIs are a minimal emulation of the Python 2 C API for built-in file -objects, which used to rely on the buffered I/O (:c:type:`FILE\*`) support +objects, which used to rely on the buffered I/O (:c:type:`FILE*`) support from the C standard library. In Python 3, files and streams use the new :mod:`io` module, which defines several layers over the low-level unbuffered I/O of the operating system. The functions described below are @@ -17,13 +17,13 @@ error reporting in the interpreter; third-party code is advised to access the :mod:`io` APIs instead. -.. c:function:: PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd) +.. c:function:: PyObject* PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd) Create a Python file object from the file descriptor of an already opened file *fd*. The arguments *name*, *encoding*, *errors* and *newline* - can be *NULL* to use the defaults; *buffering* can be *-1* to use the + can be ``NULL`` to use the defaults; *buffering* can be *-1* to use the default. *name* is ignored and kept for backward compatibility. Return - *NULL* on failure. For a more comprehensive description of the arguments, + ``NULL`` on failure. For a more comprehensive description of the arguments, please refer to the :func:`io.open` function documentation. .. warning:: @@ -60,6 +60,32 @@ the :mod:`io` APIs instead. raised if the end of the file is reached immediately. +.. c:function:: int PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction handler) + + Overrides the normal behavior of :func:`io.open_code` to pass its parameter + through the provided handler. + + The handler is a function of type :c:type:`PyObject *(\*)(PyObject *path, + void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`. + + The *userData* pointer is passed into the hook function. Since hook + functions may be called from different runtimes, this pointer should not + refer directly to Python state. + + As this hook is intentionally used during import, avoid importing new modules + during its execution unless they are known to be frozen or available in + ``sys.modules``. + + Once a hook has been set, it cannot be removed or replaced, and later calls to + :c:func:`PyFile_SetOpenCodeHook` will fail. On failure, the function returns + -1 and sets an exception if the interpreter has been initialized. + + This function is safe to call before :c:func:`Py_Initialize`. + + .. versionadded:: 3.8 + + + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) .. index:: single: Py_PRINT_RAW diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 27a75e3e0c2eeb..b29937dbecdcfd 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _floatobjects: @@ -34,12 +34,12 @@ Floating Point Objects .. c:function:: PyObject* PyFloat_FromString(PyObject *str) Create a :c:type:`PyFloatObject` object based on the string value in *str*, or - *NULL* on failure. + ``NULL`` on failure. .. c:function:: PyObject* PyFloat_FromDouble(double v) - Create a :c:type:`PyFloatObject` object from *v*, or *NULL* on failure. + Create a :c:type:`PyFloatObject` object from *v*, or ``NULL`` on failure. .. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) @@ -47,9 +47,13 @@ Floating Point Objects Return a C :c:type:`double` representation of the contents of *pyfloat*. If *pyfloat* is not a Python floating point object but has a :meth:`__float__` method, this method will first be called to convert *pyfloat* into a float. + If ``__float__()`` is not defined then it falls back to :meth:`__index__`. This method returns ``-1.0`` upon failure, so one should call :c:func:`PyErr_Occurred` to check for errors. + .. versionchanged:: 3.8 + Use :meth:`__index__` if available. + .. c:function:: double PyFloat_AS_DOUBLE(PyObject *pyfloat) @@ -72,8 +76,3 @@ Floating Point Objects .. c:function:: double PyFloat_GetMin() Return the minimum normalized positive float *DBL_MIN* as C :c:type:`double`. - -.. c:function:: int PyFloat_ClearFreeList() - - Clear the float free list. Return the number of items that could not - be freed. diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 17279c732ae076..bb416f4bb63aa2 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _function-objects: @@ -26,7 +26,7 @@ There are a few functions specific to Python functions. .. c:function:: int PyFunction_Check(PyObject *o) Return true if *o* is a function object (has type :c:data:`PyFunction_Type`). - The parameter must not be *NULL*. + The parameter must not be ``NULL``. .. c:function:: PyObject* PyFunction_New(PyObject *code, PyObject *globals) @@ -36,14 +36,14 @@ There are a few functions specific to Python functions. The function's docstring and name are retrieved from the code object. *__module__* is retrieved from *globals*. The argument defaults, annotations and closure are - set to *NULL*. *__qualname__* is set to the same value as the function's name. + set to ``NULL``. *__qualname__* is set to the same value as the function's name. .. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) As :c:func:`PyFunction_New`, but also allows setting the function object's - ``__qualname__`` attribute. *qualname* should be a unicode object or NULL; - if NULL, the ``__qualname__`` attribute is set to the same value as its + ``__qualname__`` attribute. *qualname* should be a unicode object or ``NULL``; + if ``NULL``, the ``__qualname__`` attribute is set to the same value as its ``__name__`` attribute. .. versionadded:: 3.3 @@ -69,27 +69,27 @@ There are a few functions specific to Python functions. .. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) Return the argument default values of the function object *op*. This can be a - tuple of arguments or *NULL*. + tuple of arguments or ``NULL``. .. c:function:: int PyFunction_SetDefaults(PyObject *op, PyObject *defaults) Set the argument default values for the function object *op*. *defaults* must be - *Py_None* or a tuple. + ``Py_None`` or a tuple. Raises :exc:`SystemError` and returns ``-1`` on failure. .. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) - Return the closure associated with the function object *op*. This can be *NULL* + Return the closure associated with the function object *op*. This can be ``NULL`` or a tuple of cell objects. .. c:function:: int PyFunction_SetClosure(PyObject *op, PyObject *closure) Set the closure associated with the function object *op*. *closure* must be - *Py_None* or a tuple of cell objects. + ``Py_None`` or a tuple of cell objects. Raises :exc:`SystemError` and returns ``-1`` on failure. @@ -97,12 +97,12 @@ There are a few functions specific to Python functions. .. c:function:: PyObject *PyFunction_GetAnnotations(PyObject *op) Return the annotations of the function object *op*. This can be a - mutable dictionary or *NULL*. + mutable dictionary or ``NULL``. .. c:function:: int PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) Set the annotations for the function object *op*. *annotations* - must be a dictionary or *Py_None*. + must be a dictionary or ``Py_None``. Raises :exc:`SystemError` and returns ``-1`` on failure. diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index b739aacea8d385..eee114c19d5904 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _supporting-cycle-detection: @@ -49,7 +49,7 @@ Constructors for container types must conform to two rules: .. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) Resize an object allocated by :c:func:`PyObject_NewVar`. Returns the - resized object or *NULL* on failure. *op* must not be tracked by the collector yet. + resized object or ``NULL`` on failure. *op* must not be tracked by the collector yet. .. c:function:: void PyObject_GC_Track(PyObject *op) @@ -61,6 +61,33 @@ Constructors for container types must conform to two rules: end of the constructor. +.. c:function:: int PyObject_IS_GC(PyObject *obj) + + Returns non-zero if the object implements the garbage collector protocol, + otherwise returns 0. + + The object cannot be tracked by the garbage collector if this function returns 0. + + +.. c:function:: int PyObject_GC_IsTracked(PyObject *op) + + Returns 1 if the object type of *op* implements the GC protocol and *op* is being + currently tracked by the garbage collector and 0 otherwise. + + This is analogous to the Python function :func:`gc.is_tracked`. + + .. versionadded:: 3.9 + + +.. c:function:: int PyObject_GC_IsFinalized(PyObject *op) + + Returns 1 if the object type of *op* implements the GC protocol and *op* has been + already finalized by the garbage collector and 0 otherwise. + + This is analogous to the Python function :func:`gc.is_finalized`. + + .. versionadded:: 3.9 + Similarly, the deallocator for the object must conform to a similar pair of rules: @@ -110,7 +137,7 @@ The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: Traversal function for a container object. Implementations must call the *visit* function for each object directly contained by *self*, with the parameters to *visit* being the contained object and the *arg* value passed - to the handler. The *visit* function must not be called with a *NULL* + to the handler. The *visit* function must not be called with a ``NULL`` object argument. If *visit* returns a non-zero value that value should be returned immediately. @@ -121,7 +148,7 @@ must name its arguments exactly *visit* and *arg*: .. c:function:: void Py_VISIT(PyObject *o) - If *o* is not *NULL*, call the *visit* callback, with arguments *o* + If *o* is not ``NULL``, call the *visit* callback, with arguments *o* and *arg*. If *visit* returns a non-zero value, then return it. Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers look like:: @@ -134,7 +161,7 @@ must name its arguments exactly *visit* and *arg*: return 0; } -The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or *NULL* +The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or ``NULL`` if the object is immutable. diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 1efbae4fcba098..e098425e6364d9 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _gen-objects: @@ -15,6 +15,11 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. The C structure used for generator objects. +.. c:type:: PySendResult + + The enum value used to represent different results of :c:func:`PyGen_Send`. + + .. c:var:: PyTypeObject PyGen_Type The type object corresponding to generator objects. @@ -22,23 +27,33 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. .. c:function:: int PyGen_Check(PyObject *ob) - Return true if *ob* is a generator object; *ob* must not be *NULL*. + Return true if *ob* is a generator object; *ob* must not be ``NULL``. .. c:function:: int PyGen_CheckExact(PyObject *ob) - Return true if *ob*'s type is *PyGen_Type*; *ob* must not be *NULL*. + Return true if *ob*'s type is :c:type:`PyGen_Type`; *ob* must not be ``NULL``. .. c:function:: PyObject* PyGen_New(PyFrameObject *frame) Create and return a new generator object based on the *frame* object. A reference to *frame* is stolen by this function. The argument must not be - *NULL*. + ``NULL``. .. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname) Create and return a new generator object based on the *frame* object, with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument - must not be *NULL*. + must not be ``NULL``. + +.. c:function:: PySendResult PyGen_Send(PyGenObject *gen, PyObject *arg, PyObject **presult) + + Sends the *arg* value into the generator *gen*. Coroutine objects + are also allowed to be as the *gen* argument but they need to be + explicitly casted to PyGenObject*. Returns: + + - ``PYGEN_RETURN`` if generator returns. Return value is returned via *presult*. + - ``PYGEN_NEXT`` if generator yields. Yielded value is returned via *presult*. + - ``PYGEN_ERROR`` if generator has raised and exception. *presult* is set to ``NULL``. diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8cdc256e7c9e0e..c6fc33076f0f51 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _importing: @@ -14,7 +14,7 @@ Importing Modules single: modules (in module sys) This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below, - leaving the *globals* and *locals* arguments set to *NULL* and *level* set + leaving the *globals* and *locals* arguments set to ``NULL`` and *level* set to 0. When the *name* argument contains a dot (when it specifies a submodule of a package), the *fromlist* argument is set to the list ``['*']`` so that the return value is the @@ -22,7 +22,7 @@ Importing Modules be the case. (Unfortunately, this has an additional side effect when *name* in fact specifies a subpackage instead of a submodule: the submodules specified in the package's ``__all__`` variable are loaded.) Return a new reference to the - imported module, or *NULL* with an exception set on failure. A failing + imported module, or ``NULL`` with an exception set on failure. A failing import of a module doesn't leave the module in :data:`sys.modules`. This function always uses absolute imports. @@ -47,7 +47,7 @@ Importing Modules function :func:`__import__`. The return value is a new reference to the imported module or top-level - package, or *NULL* with an exception set on failure. Like for + package, or ``NULL`` with an exception set on failure. Like for :func:`__import__`, the return value when a submodule of a package was requested is normally the top-level package, unless a non-empty *fromlist* was given. @@ -63,7 +63,7 @@ Importing Modules this function directly. The return value is a new reference to the imported module or top-level package, - or *NULL* with an exception set on failure. Like for :func:`__import__`, + or ``NULL`` with an exception set on failure. Like for :func:`__import__`, the return value when a submodule of a package was requested is normally the top-level package, unless a non-empty *fromlist* was given. @@ -91,7 +91,7 @@ Importing Modules .. c:function:: PyObject* PyImport_ReloadModule(PyObject *m) - Reload a module. Return a new reference to the reloaded module, or *NULL* with + Reload a module. Return a new reference to the reloaded module, or ``NULL`` with an exception set on failure (the module still exists in this case). @@ -100,7 +100,7 @@ Importing Modules Return the module object corresponding to a module name. The *name* argument may be of the form ``package.module``. First check the modules dictionary if there's one there, and if not, create a new one and insert it in the modules - dictionary. Return *NULL* with an exception set on failure. + dictionary. Return ``NULL`` with an exception set on failure. .. note:: @@ -125,7 +125,7 @@ Importing Modules Given a module name (possibly of the form ``package.module``) and a code object read from a Python bytecode file or obtained from the built-in function :func:`compile`, load the module. Return a new reference to the module object, - or *NULL* with an exception set if an error occurred. *name* + or ``NULL`` with an exception set if an error occurred. *name* is removed from :attr:`sys.modules` in error cases, even if *name* was already in :attr:`sys.modules` on entry to :c:func:`PyImport_ExecCodeModule`. Leaving incompletely initialized modules in :attr:`sys.modules` is dangerous, as imports of @@ -207,8 +207,8 @@ Importing Modules .. c:function:: PyObject* PyImport_GetModule(PyObject *name) Return the already imported module with the given name. If the - module has not been imported yet then returns NULL but does not set - an error. Returns NULL and sets an error if the lookup failed. + module has not been imported yet then returns ``NULL`` but does not set + an error. Returns ``NULL`` and sets an error if the lookup failed. .. versionadded:: 3.7 @@ -223,21 +223,6 @@ Importing Modules Return a new reference to the finder object. -.. c:function:: void _PyImport_Init() - - Initialize the import mechanism. For internal use only. - - -.. c:function:: void PyImport_Cleanup() - - Empty the module table. For internal use only. - - -.. c:function:: void _PyImport_Fini() - - Finalize the import mechanism. For internal use only. - - .. c:function:: int PyImport_ImportFrozenModuleObject(PyObject *name) Load a frozen module named *name*. Return ``1`` for success, ``0`` if the @@ -277,7 +262,7 @@ Importing Modules .. c:var:: const struct _frozen* PyImport_FrozenModules This pointer is initialized to point to an array of :c:type:`struct _frozen` - records, terminated by one whose members are all *NULL* or zero. When a frozen + records, terminated by one whose members are all ``NULL`` or zero. When a frozen module is imported, it is searched in this table. Third-party code could play tricks with this to provide a dynamically created collection of frozen modules. @@ -310,7 +295,7 @@ Importing Modules .. c:function:: int PyImport_ExtendInittab(struct _inittab *newtab) Add a collection of modules to the table of built-in modules. The *newtab* - array must end with a sentinel entry which contains *NULL* for the :attr:`name` + array must end with a sentinel entry which contains ``NULL`` for the :attr:`name` field; failure to provide the sentinel value can result in a memory fault. Returns ``0`` on success or ``-1`` if insufficient memory could be allocated to extend the internal table. In the event of failure, no modules are added to the diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index 3bfbaf4eedde0e..9a8f1507b3f4cc 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -21,6 +21,7 @@ document the API functions in detail. abstract.rst concrete.rst init.rst + init_config.rst memory.rst objimpl.rst apiabiversion.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 7ef11228a33dcf..7f06648bcb4572 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _initialization: @@ -7,6 +7,8 @@ Initialization, Finalization, and Threads ***************************************** +See also :ref:`Python Initialization Configuration `. + .. _pre-init-safe: Before Python Initialization @@ -79,7 +81,7 @@ When a flag is set by an option, the value of the flag is the number of times that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. -.. c:var:: Py_BytesWarningFlag +.. c:var:: int Py_BytesWarningFlag Issue a warning when comparing :class:`bytes` or :class:`bytearray` with :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater @@ -87,7 +89,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-b` option. -.. c:var:: Py_DebugFlag +.. c:var:: int Py_DebugFlag Turn on parser debugging output (for expert only, depending on compilation options). @@ -95,7 +97,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment variable. -.. c:var:: Py_DontWriteBytecodeFlag +.. c:var:: int Py_DontWriteBytecodeFlag If set to non-zero, Python won't try to write ``.pyc`` files on the import of source modules. @@ -103,14 +105,14 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. -.. c:var:: Py_FrozenFlag +.. c:var:: int Py_FrozenFlag Suppress error messages when calculating the module search path in :c:func:`Py_GetPath`. Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs. -.. c:var:: Py_HashRandomizationFlag +.. c:var:: int Py_HashRandomizationFlag Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to a non-empty string. @@ -118,14 +120,14 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment variable to initialize the secret hash seed. -.. c:var:: Py_IgnoreEnvironmentFlag +.. c:var:: int Py_IgnoreEnvironmentFlag Ignore all :envvar:`PYTHON*` environment variables, e.g. :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. Set by the :option:`-E` and :option:`-I` options. -.. c:var:: Py_InspectFlag +.. c:var:: int Py_InspectFlag When a script is passed as first argument or the :option:`-c` option is used, enter interactive mode after executing the script or the command, even when @@ -134,11 +136,11 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment variable. -.. c:var:: Py_InteractiveFlag +.. c:var:: int Py_InteractiveFlag Set by the :option:`-i` option. -.. c:var:: Py_IsolatedFlag +.. c:var:: int Py_IsolatedFlag Run Python in isolated mode. In isolated mode :data:`sys.path` contains neither the script's directory nor the user's site-packages directory. @@ -147,7 +149,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.4 -.. c:var:: Py_LegacyWindowsFSEncodingFlag +.. c:var:: int Py_LegacyWindowsFSEncodingFlag If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8 encoding for the filesystem encoding. @@ -159,7 +161,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. -.. c:var:: Py_LegacyWindowsStdioFlag +.. c:var:: int Py_LegacyWindowsStdioFlag If the flag is non-zero, use :class:`io.FileIO` instead of :class:`WindowsConsoleIO` for :mod:`sys` standard streams. @@ -171,7 +173,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. -.. c:var:: Py_NoSiteFlag +.. c:var:: int Py_NoSiteFlag Disable the import of the module :mod:`site` and the site-dependent manipulations of :data:`sys.path` that it entails. Also disable these @@ -180,7 +182,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-S` option. -.. c:var:: Py_NoUserSiteDirectory +.. c:var:: int Py_NoUserSiteDirectory Don't add the :data:`user site-packages directory ` to :data:`sys.path`. @@ -188,12 +190,12 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-s` and :option:`-I` options, and the :envvar:`PYTHONNOUSERSITE` environment variable. -.. c:var:: Py_OptimizeFlag +.. c:var:: int Py_OptimizeFlag Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment variable. -.. c:var:: Py_QuietFlag +.. c:var:: int Py_QuietFlag Don't display the copyright and version messages even in interactive mode. @@ -201,14 +203,14 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.2 -.. c:var:: Py_UnbufferedStdioFlag +.. c:var:: int Py_UnbufferedStdioFlag Force the stdout and stderr streams to be unbuffered. Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` environment variable. -.. c:var:: Py_VerboseFlag +.. c:var:: int Py_VerboseFlag Print a message each time a module is initialized, showing the place (filename or built-in module) from which it is loaded. If greater or equal @@ -299,8 +301,9 @@ Initializing and finalizing the interpreter than once; this can happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` more than once. - .. versionadded:: 3.6 + .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx + .. versionadded:: 3.6 .. c:function:: void Py_Finalize() @@ -326,7 +329,7 @@ Process-wide parameters It overrides :envvar:`PYTHONIOENCODING` values, and allows embedding code to control IO encoding when the environment variable does not work. - ``encoding`` and/or ``errors`` may be NULL to use + *encoding* and/or *errors* may be ``NULL`` to use :envvar:`PYTHONIOENCODING` and/or default values (depending on other settings). @@ -469,8 +472,8 @@ Process-wide parameters dependent delimiter character, which is ``':'`` on Unix and Mac OS X, ``';'`` on Windows. - This also causes :data:`sys.executable` to be set only to the raw program - name (see :c:func:`Py_SetProgramName`) and for :data:`sys.prefix` and + This also causes :data:`sys.executable` to be set to the program + full path (see :c:func:`Py_GetProgramFullPath`) and for :data:`sys.prefix` and :data:`sys.exec_prefix` to be empty. It is up to the caller to modify these if required after calling :c:func:`Py_Initialize`. @@ -480,6 +483,10 @@ Process-wide parameters The path argument is copied internally, so the caller may free it after the call completes. + .. versionchanged:: 3.8 + The program full path is now used for :data:`sys.executable`, instead + of the program name. + .. c:function:: const char* Py_GetVersion() @@ -762,9 +769,19 @@ supports the creation of additional interpreters (using :c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the :c:func:`PyGILState_\*` API is unsupported. + +.. _fork-and-threads: + +Cautions about fork() +--------------------- + Another important thing to note about threads is their behaviour in the face of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. That also +process forks only the thread that issued the fork will exist. This has a +concrete impact both on how locks must be handled and on all stored state +in CPython's runtime. + +The fact that only the "current" thread remains means any locks held by other threads will never be released. Python solves this for :func:`os.fork` by acquiring the locks it uses internally before the fork, and releasing them afterwards. In addition, it resets any @@ -779,6 +796,17 @@ being held by a thread that is defunct after the fork. :c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not always able to. +The fact that all other threads go away also means that CPython's +runtime state there must be cleaned up properly, which :func:`os.fork` +does. This means finalizing all other :c:type:`PyThreadState` objects +belonging to the current interpreter and all other +:c:type:`PyInterpreterState` objects. Due to this and the special +nature of the :ref:`"main" interpreter `, +:c:func:`fork` should only be called in that interpreter's "main" +thread, where the CPython global runtime was originally initialized. +The only exception is if :c:func:`exec` will be called immediately +after. + High-level API -------------- @@ -802,7 +830,7 @@ code, or when embedding the Python interpreter: .. c:type:: PyThreadState This data structure represents the state of a single thread. The only public - data member is :c:type:`PyInterpreterState \*`:attr:`interp`, which points to + data member is :attr:`interp` (:c:type:`PyInterpreterState *`), which points to this thread's interpreter state. @@ -814,12 +842,12 @@ code, or when embedding the Python interpreter: single: PyEval_SaveThread() single: PyEval_RestoreThread() - Initialize and acquire the global interpreter lock. It should be called in the - main thread before creating a second thread or engaging in any other thread - operations such as ``PyEval_ReleaseThread(tstate)``. It is not needed before - calling :c:func:`PyEval_SaveThread` or :c:func:`PyEval_RestoreThread`. + Deprecated function which does nothing. - This is a no-op when called for a second time. + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. .. versionchanged:: 3.7 This function is now called by :c:func:`Py_Initialize()`, so you don't @@ -828,6 +856,8 @@ code, or when embedding the Python interpreter: .. versionchanged:: 3.2 This function cannot be called before :c:func:`Py_Initialize()` anymore. + .. deprecated-removed:: 3.9 3.11 + .. index:: module: _thread @@ -840,19 +870,21 @@ code, or when embedding the Python interpreter: .. versionchanged:: 3.7 The :term:`GIL` is now initialized by :c:func:`Py_Initialize()`. + .. deprecated-removed:: 3.9 3.11 + .. c:function:: PyThreadState* PyEval_SaveThread() Release the global interpreter lock (if it has been created) and reset the - thread state to *NULL*, returning the previous thread state (which is not - *NULL*). If the lock has been created, the current thread must have + thread state to ``NULL``, returning the previous thread state (which is not + ``NULL``). If the lock has been created, the current thread must have acquired it. .. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) Acquire the global interpreter lock (if it has been created) and set the - thread state to *tstate*, which must not be *NULL*. If the lock has been + thread state to *tstate*, which must not be ``NULL``. If the lock has been created, the current thread must not have acquired it, otherwise deadlock ensues. @@ -866,24 +898,17 @@ code, or when embedding the Python interpreter: .. c:function:: PyThreadState* PyThreadState_Get() Return the current thread state. The global interpreter lock must be held. - When the current thread state is *NULL*, this issues a fatal error (so that - the caller needn't check for *NULL*). + When the current thread state is ``NULL``, this issues a fatal error (so that + the caller needn't check for ``NULL``). .. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) Swap the current thread state with the thread state given by the argument - *tstate*, which may be *NULL*. The global interpreter lock must be held + *tstate*, which may be ``NULL``. The global interpreter lock must be held and is not released. -.. c:function:: void PyEval_ReInitThreads() - - This function is called from :c:func:`PyOS_AfterFork_Child` to ensure - that newly created child processes don't hold locks referring to threads - which are not running in the child process. - - The following functions use thread-local storage, and are not compatible with sub-interpreters: @@ -997,12 +1022,16 @@ All of the following functions must be called after :c:func:`Py_Initialize`. be held, but may be held if it is necessary to serialize calls to this function. + .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New + .. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) Reset all information in an interpreter state object. The global interpreter lock must be held. + .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear + .. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) @@ -1023,6 +1052,10 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Reset all information in a thread state object. The global interpreter lock must be held. + .. versionchanged:: 3.9 + This function now calls the :c:member:`PyThreadState.on_delete` callback. + Previously, that happened in :c:func:`PyThreadState_Delete`. + .. c:function:: void PyThreadState_Delete(PyThreadState *tstate) @@ -1031,18 +1064,72 @@ All of the following functions must be called after :c:func:`Py_Initialize`. :c:func:`PyThreadState_Clear`. -.. c:function:: PY_INT64_T PyInterpreterState_GetID(PyInterpreterState *interp) +.. c:function:: void PyThreadState_DeleteCurrent(void) + + Destroy the current thread state and release the global interpreter lock. + Like :c:func:`PyThreadState_Delete`, the global interpreter lock need not + be held. The thread state must have been reset with a previous call + to :c:func:`PyThreadState_Clear`. + + +.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) + + Get the current frame of the Python thread state *tstate*. + + Return a strong reference. Return ``NULL`` if no frame is currently + executing. + + See also :c:func:`PyEval_GetFrame`. + + *tstate* must not be ``NULL``. + + .. versionadded:: 3.9 + + +.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) + + Get the unique thread state identifier of the Python thread state *tstate*. + + *tstate* must not be ``NULL``. + + .. versionadded:: 3.9 + + +.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) + + Get the interpreter of the Python thread state *tstate*. + + *tstate* must not be ``NULL``. + + .. versionadded:: 3.9 + + +.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) + + Get the current interpreter. + + Issue a fatal error if there no current Python thread state or no current + interpreter. It cannot return NULL. + + The caller must hold the GIL. + + .. versionadded:: 3.9 + + +.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) Return the interpreter's unique ID. If there was any error in doing so then ``-1`` is returned and an error is set. + The caller must hold the GIL. + .. versionadded:: 3.7 .. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) Return a dictionary in which interpreter-specific data may be stored. - If this function returns *NULL* then no exception has been raised and + If this function returns ``NULL`` then no exception has been raised and the caller should assume no interpreter-specific dict is available. This is not a replacement for :c:func:`PyModule_GetState()`, which @@ -1050,13 +1137,39 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.8 +.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, PyFrameObject *frame, int throwflag) + + Type of a frame evaluation function. + + The *throwflag* parameter is used by the ``throw()`` method of generators: + if non-zero, handle the current exception. + + .. versionchanged:: 3.9 + The function now takes a *tstate* parameter. + +.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) + + Get the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + +.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); + + Set the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + .. c:function:: PyObject* PyThreadState_GetDict() Return a dictionary in which extensions can store thread-specific state information. Each extension should use a unique key to use to store state in the dictionary. It is okay to call this function when no current thread state - is available. If this function returns *NULL*, no exception has been raised and + is available. If this function returns ``NULL``, no exception has been raised and the caller should assume no current thread state is available. @@ -1077,18 +1190,30 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) Acquire the global interpreter lock and set the current thread state to - *tstate*, which should not be *NULL*. The lock must have been created earlier. + *tstate*, which must not be ``NULL``. The lock must have been created earlier. If this thread already has the lock, deadlock ensues. + .. note:: + Calling this function from a thread when the runtime is finalizing + will terminate the thread, even if the thread was not created by Python. + You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to + check if the interpreter is in process of being finalized before calling + this function to avoid unwanted termination. + + .. versionchanged:: 3.8 + Updated to be consistent with :c:func:`PyEval_RestoreThread`, + :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, + and terminate the current thread if called while the interpreter is finalizing. + :c:func:`PyEval_RestoreThread` is a higher-level function which is always available (even when threads have not been initialized). .. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - Reset the current thread state to *NULL* and release the global interpreter + Reset the current thread state to ``NULL`` and release the global interpreter lock. The lock must have been created earlier and must be held by the current - thread. The *tstate* argument, which must not be *NULL*, is only used to check + thread. The *tstate* argument, which must not be ``NULL``, is only used to check that it represents the current thread state --- if it isn't, a fatal error is reported. @@ -1106,6 +1231,18 @@ All of the following functions must be called after :c:func:`Py_Initialize`. :c:func:`PyEval_RestoreThread` or :c:func:`PyEval_AcquireThread` instead. + .. note:: + Calling this function from a thread when the runtime is finalizing + will terminate the thread, even if the thread was not created by Python. + You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to + check if the interpreter is in process of being finalized before calling + this function to avoid unwanted termination. + + .. versionchanged:: 3.8 + Updated to be consistent with :c:func:`PyEval_RestoreThread`, + :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, + and terminate the current thread if called while the interpreter is finalizing. + .. c:function:: void PyEval_ReleaseLock() @@ -1124,10 +1261,18 @@ Sub-interpreter support While in most uses, you will only embed a single Python interpreter, there are cases where you need to create several independent interpreters in the -same process and perhaps even in the same thread. Sub-interpreters allow -you to do that. You can switch between sub-interpreters using the -:c:func:`PyThreadState_Swap` function. You can create and destroy them -using the following functions: +same process and perhaps even in the same thread. Sub-interpreters allow +you to do that. + +The "main" interpreter is the first one created when the runtime initializes. +It is usually the only Python interpreter in a process. Unlike sub-interpreters, +the main interpreter has unique process-global responsibilities like signal +handling. It is also responsible for execution during runtime initialization and +is usually the active interpreter during runtime finalization. The +:c:func:`PyInterpreterState_Main` function returns a pointer to its state. + +You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` +function. You can create and destroy them using the following functions: .. c:function:: PyThreadState* Py_NewInterpreter() @@ -1153,7 +1298,7 @@ using the following functions: The return value points to the first thread state created in the new sub-interpreter. This thread state is made in the current thread state. Note that no actual thread is created; see the discussion of thread states - below. If creation of the new interpreter is unsuccessful, *NULL* is + below. If creation of the new interpreter is unsuccessful, ``NULL`` is returned; no exception is set since the exception state is stored in the current thread state and there may not be a current thread state. (Like all other Python/C API functions, the global interpreter lock must be held before @@ -1165,15 +1310,31 @@ using the following functions: single: Py_FinalizeEx() single: Py_Initialize() - Extension modules are shared between (sub-)interpreters as follows: the first - time a particular extension is imported, it is initialized normally, and a - (shallow) copy of its module's dictionary is squirreled away. When the same - extension is imported by another (sub-)interpreter, a new module is initialized - and filled with the contents of this copy; the extension's ``init`` function is - not called. Note that this is different from what happens when an extension is - imported after the interpreter has been completely re-initialized by calling - :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that case, the extension's - ``initmodule`` function *is* called again. + Extension modules are shared between (sub-)interpreters as follows: + + * For modules using multi-phase initialization, + e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is + created and initialized for each interpreter. + Only C-level static and global variables are shared between these + module objects. + + * For modules using single-phase initialization, + e.g. :c:func:`PyModule_Create`, the first time a particular extension + is imported, it is initialized normally, and a (shallow) copy of its + module's dictionary is squirreled away. + When the same extension is imported by another (sub-)interpreter, a new + module is initialized and filled with the contents of this copy; the + extension's ``init`` function is not called. + Objects in the module's dictionary thus end up shared across + (sub-)interpreters, which might cause unwanted behavior (see + `Bugs and caveats`_ below). + + Note that this is different from what happens when an extension is + imported after the interpreter has been completely re-initialized by + calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that + case, the extension's ``initmodule`` function *is* called again. + As with multi-phase initialization, this means that only C-level static + and global variables are shared between these modules. .. index:: single: close() (in module os) @@ -1184,7 +1345,7 @@ using the following functions: Destroy the (sub-)interpreter represented by the given thread state. The given thread state must be the current thread state. See the discussion of thread - states below. When the call returns, the current thread state is *NULL*. All + states below. When the call returns, the current thread state is ``NULL``. All thread states associated with this interpreter are destroyed. (The global interpreter lock must be held before calling this function and is still held when it returns.) :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that @@ -1199,14 +1360,16 @@ process, the insulation between them isn't perfect --- for example, using low-level file operations like :func:`os.close` they can (accidentally or maliciously) affect each other's open files. Because of the way extensions are shared between (sub-)interpreters, some extensions may not -work properly; this is especially likely when the extension makes use of -(static) global variables, or when the extension manipulates its module's -dictionary after its initialization. It is possible to insert objects created -in one sub-interpreter into a namespace of another sub-interpreter; this should -be done with great care to avoid sharing user-defined functions, methods, -instances or classes between sub-interpreters, since import operations executed -by such objects may affect the wrong (sub-)interpreter's dictionary of loaded -modules. +work properly; this is especially likely when using single-phase initialization +or (static) global variables. +It is possible to insert objects created in one sub-interpreter into +a namespace of another (sub-)interpreter; this should be avoided if possible. + +Special care should be taken to avoid sharing user-defined functions, +methods, instances or classes between sub-interpreters, since import +operations executed by such objects may affect the wrong (sub-)interpreter's +dictionary of loaded modules. It is equally important to avoid sharing +objects from which the above are reachable. Also note that combining this functionality with :c:func:`PyGILState_\*` APIs is delicate, because these APIs assume a bijection between Python thread states @@ -1251,6 +1414,10 @@ pointer and a void pointer argument. This function doesn't need a current thread state to run, and it doesn't need the global interpreter lock. + To call this function in a subinterpreter, the caller must hold the GIL. + Otherwise, the function *func* can be scheduled to be called from the wrong + interpreter. + .. warning:: This is a low-level function, only useful for very special cases. There is no guarantee that *func* will be called as quick as @@ -1259,6 +1426,12 @@ pointer and a void pointer argument. function is generally **not** suitable for calling Python code from arbitrary C threads. Instead, use the :ref:`PyGILState API`. + .. versionchanged:: 3.9 + If this function is called in a subinterpreter, the function *func* is + now scheduled to be called from the subinterpreter, rather than being + called from the main interpreter. Each subinterpreter now has its own + list of scheduled calls. + .. versionadded:: 3.1 .. _profiling: @@ -1291,27 +1464,27 @@ Python-level trace functions in previous versions. :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION`, :const:`PyTrace_C_RETURN`, or :const:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: - +------------------------------+--------------------------------------+ - | Value of *what* | Meaning of *arg* | - +==============================+======================================+ - | :const:`PyTrace_CALL` | Always :c:data:`Py_None`. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_EXCEPTION` | Exception information as returned by | - | | :func:`sys.exc_info`. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_LINE` | Always :c:data:`Py_None`. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_RETURN` | Value being returned to the caller, | - | | or *NULL* if caused by an exception. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_C_CALL` | Function object being called. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_C_EXCEPTION` | Function object being called. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_C_RETURN` | Function object being called. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | - +------------------------------+--------------------------------------+ + +------------------------------+----------------------------------------+ + | Value of *what* | Meaning of *arg* | + +==============================+========================================+ + | :const:`PyTrace_CALL` | Always :c:data:`Py_None`. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_EXCEPTION` | Exception information as returned by | + | | :func:`sys.exc_info`. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_LINE` | Always :c:data:`Py_None`. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_RETURN` | Value being returned to the caller, | + | | or ``NULL`` if caused by an exception. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_C_CALL` | Function object being called. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_C_EXCEPTION` | Function object being called. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_C_RETURN` | Function object being called. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | + +------------------------------+----------------------------------------+ .. c:var:: int PyTrace_CALL @@ -1375,12 +1548,14 @@ Python-level trace functions in previous versions. .. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) Set the profiler function to *func*. The *obj* parameter is passed to the - function as its first parameter, and may be any Python object, or *NULL*. If + function as its first parameter, and may be any Python object, or ``NULL``. If the profile function needs to maintain state, using a different value for *obj* for each thread provides a convenient and thread-safe place to store it. The profile function is called for all monitored events except :const:`PyTrace_LINE` :const:`PyTrace_OPCODE` and :const:`PyTrace_EXCEPTION`. + The caller must hold the :term:`GIL`. + .. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) @@ -1391,6 +1566,9 @@ Python-level trace functions in previous versions. will not receive :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or :const:`PyTrace_C_RETURN` as a value for the *what* parameter. + The caller must hold the :term:`GIL`. + + .. _advanced-debugging: Advanced Debugger Support @@ -1441,7 +1619,7 @@ The Python interpreter provides low-level support for thread-local storage (TLS) which wraps the underlying native TLS implementation to support the Python-level thread local storage API (:class:`threading.local`). The CPython C level APIs are similar to those offered by pthreads and Windows: -use a thread key and functions to associate a :c:type:`void\*` value per +use a thread key and functions to associate a :c:type:`void*` value per thread. The GIL does *not* need to be held when calling these functions; they supply @@ -1452,8 +1630,8 @@ you need to include :file:`pythread.h` to use thread-local storage. .. note:: None of these API functions handle memory management on behalf of the - :c:type:`void\*` values. You need to allocate and deallocate them yourself. - If the :c:type:`void\*` values happen to be :c:type:`PyObject\*`, these + :c:type:`void*` values. You need to allocate and deallocate them yourself. + If the :c:type:`void*` values happen to be :c:type:`PyObject*`, these functions don't do refcount operations on them either. .. _thread-specific-storage-api: @@ -1498,7 +1676,7 @@ is not possible due to its implementation being opaque at build time. .. c:function:: Py_tss_t* PyThread_tss_alloc() Return a value which is the same state as a value initialized with - :c:macro:`Py_tss_NEEDS_INIT`, or *NULL* in the case of dynamic allocation + :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation failure. @@ -1517,7 +1695,7 @@ is not possible due to its implementation being opaque at build time. Methods ~~~~~~~ -The parameter *key* of these functions must not be *NULL*. Moreover, the +The parameter *key* of these functions must not be ``NULL``. Moreover, the behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are undefined if the given :c:type:`Py_tss_t` has not been initialized by :c:func:`PyThread_tss_create`. @@ -1549,15 +1727,15 @@ undefined if the given :c:type:`Py_tss_t` has not been initialized by .. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) - Return a zero value to indicate successfully associating a :c:type:`void\*` + Return a zero value to indicate successfully associating a :c:type:`void*` value with a TSS key in the current thread. Each thread has a distinct - mapping of the key to a :c:type:`void\*` value. + mapping of the key to a :c:type:`void*` value. .. c:function:: void* PyThread_tss_get(Py_tss_t *key) - Return the :c:type:`void\*` value associated with a TSS key in the current - thread. This returns *NULL* if no value is associated with the key in the + Return the :c:type:`void*` value associated with a TSS key in the current + thread. This returns ``NULL`` if no value is associated with the key in the current thread. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst new file mode 100644 index 00000000000000..37f5b9f880bf14 --- /dev/null +++ b/Doc/c-api/init_config.rst @@ -0,0 +1,1093 @@ +.. highlight:: c + +.. _init-config: + +*********************************** +Python Initialization Configuration +*********************************** + +.. versionadded:: 3.8 + +Structures: + +* :c:type:`PyConfig` +* :c:type:`PyPreConfig` +* :c:type:`PyStatus` +* :c:type:`PyWideStringList` + +Functions: + +* :c:func:`PyConfig_Clear` +* :c:func:`PyConfig_InitIsolatedConfig` +* :c:func:`PyConfig_InitPythonConfig` +* :c:func:`PyConfig_Read` +* :c:func:`PyConfig_SetArgv` +* :c:func:`PyConfig_SetBytesArgv` +* :c:func:`PyConfig_SetBytesString` +* :c:func:`PyConfig_SetString` +* :c:func:`PyConfig_SetWideStringList` +* :c:func:`PyPreConfig_InitIsolatedConfig` +* :c:func:`PyPreConfig_InitPythonConfig` +* :c:func:`PyStatus_Error` +* :c:func:`PyStatus_Exception` +* :c:func:`PyStatus_Exit` +* :c:func:`PyStatus_IsError` +* :c:func:`PyStatus_IsExit` +* :c:func:`PyStatus_NoMemory` +* :c:func:`PyStatus_Ok` +* :c:func:`PyWideStringList_Append` +* :c:func:`PyWideStringList_Insert` +* :c:func:`Py_ExitStatusException` +* :c:func:`Py_InitializeFromConfig` +* :c:func:`Py_PreInitialize` +* :c:func:`Py_PreInitializeFromArgs` +* :c:func:`Py_PreInitializeFromBytesArgs` +* :c:func:`Py_RunMain` +* :c:func:`Py_GetArgcArgv` + +The preconfiguration (``PyPreConfig`` type) is stored in +``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in +``PyInterpreterState.config``. + +See also :ref:`Initialization, Finalization, and Threads `. + +.. seealso:: + :pep:`587` "Python Initialization Configuration". + + +PyWideStringList +---------------- + +.. c:type:: PyWideStringList + + List of ``wchar_t*`` strings. + + If *length* is non-zero, *items* must be non-``NULL`` and all strings must be + non-``NULL``. + + Methods: + + .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) + + Append *item* to *list*. + + Python must be preinitialized to call this function. + + .. c:function:: PyStatus PyWideStringList_Insert(PyWideStringList *list, Py_ssize_t index, const wchar_t *item) + + Insert *item* into *list* at *index*. + + If *index* is greater than or equal to *list* length, append *item* to + *list*. + + *index* must be greater than or equal to 0. + + Python must be preinitialized to call this function. + + Structure fields: + + .. c:member:: Py_ssize_t length + + List length. + + .. c:member:: wchar_t** items + + List items. + +PyStatus +-------- + +.. c:type:: PyStatus + + Structure to store an initialization function status: success, error + or exit. + + For an error, it can store the C function name which created the error. + + Structure fields: + + .. c:member:: int exitcode + + Exit code. Argument passed to ``exit()``. + + .. c:member:: const char *err_msg + + Error message. + + .. c:member:: const char *func + + Name of the function which created an error, can be ``NULL``. + + Functions to create a status: + + .. c:function:: PyStatus PyStatus_Ok(void) + + Success. + + .. c:function:: PyStatus PyStatus_Error(const char *err_msg) + + Initialization error with a message. + + .. c:function:: PyStatus PyStatus_NoMemory(void) + + Memory allocation failure (out of memory). + + .. c:function:: PyStatus PyStatus_Exit(int exitcode) + + Exit Python with the specified exit code. + + Functions to handle a status: + + .. c:function:: int PyStatus_Exception(PyStatus status) + + Is the status an error or an exit? If true, the exception must be + handled; by calling :c:func:`Py_ExitStatusException` for example. + + .. c:function:: int PyStatus_IsError(PyStatus status) + + Is the result an error? + + .. c:function:: int PyStatus_IsExit(PyStatus status) + + Is the result an exit? + + .. c:function:: void Py_ExitStatusException(PyStatus status) + + Call ``exit(exitcode)`` if *status* is an exit. Print the error + message and exit with a non-zero exit code if *status* is an error. Must + only be called if ``PyStatus_Exception(status)`` is non-zero. + +.. note:: + Internally, Python uses macros which set ``PyStatus.func``, + whereas functions to create a status set ``func`` to ``NULL``. + +Example:: + + PyStatus alloc(void **ptr, size_t size) + { + *ptr = PyMem_RawMalloc(size); + if (*ptr == NULL) { + return PyStatus_NoMemory(); + } + return PyStatus_Ok(); + } + + int main(int argc, char **argv) + { + void *ptr; + PyStatus status = alloc(&ptr, 16); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + PyMem_Free(ptr); + return 0; + } + + +PyPreConfig +----------- + +.. c:type:: PyPreConfig + + Structure used to preinitialize Python: + + * Set the Python memory allocator + * Configure the LC_CTYPE locale + * Set the UTF-8 mode + + Function to initialize a preconfiguration: + + .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) + + Initialize the preconfiguration with :ref:`Python Configuration + `. + + .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + + Initialize the preconfiguration with :ref:`Isolated Configuration + `. + + Structure fields: + + .. c:member:: int allocator + + Name of the memory allocator: + + * ``PYMEM_ALLOCATOR_NOT_SET`` (``0``): don't change memory allocators + (use defaults) + * ``PYMEM_ALLOCATOR_DEFAULT`` (``1``): default memory allocators + * ``PYMEM_ALLOCATOR_DEBUG`` (``2``): default memory allocators with + debug hooks + * ``PYMEM_ALLOCATOR_MALLOC`` (``3``): force usage of ``malloc()`` + * ``PYMEM_ALLOCATOR_MALLOC_DEBUG`` (``4``): force usage of + ``malloc()`` with debug hooks + * ``PYMEM_ALLOCATOR_PYMALLOC`` (``5``): :ref:`Python pymalloc memory + allocator ` + * ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc + memory allocator ` with debug hooks + + ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` + are not supported if Python is configured using ``--without-pymalloc`` + + See :ref:`Memory Management `. + + .. c:member:: int configure_locale + + Set the LC_CTYPE locale to the user preferred locale? If equals to 0, set + :c:member:`coerce_c_locale` and :c:member:`coerce_c_locale_warn` to 0. + + .. c:member:: int coerce_c_locale + + If equals to 2, coerce the C locale; if equals to 1, read the LC_CTYPE + locale to decide if it should be coerced. + + .. c:member:: int coerce_c_locale_warn + + If non-zero, emit a warning if the C locale is coerced. + + .. c:member:: int dev_mode + + See :c:member:`PyConfig.dev_mode`. + + .. c:member:: int isolated + + See :c:member:`PyConfig.isolated`. + + .. c:member:: int legacy_windows_fs_encoding (Windows only) + + If non-zero, disable UTF-8 Mode, set the Python filesystem encoding to + ``mbcs``, set the filesystem error handler to ``replace``. + + Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for + Windows specific code. + + .. c:member:: int parse_argv + + If non-zero, :c:func:`Py_PreInitializeFromArgs` and + :c:func:`Py_PreInitializeFromBytesArgs` parse their ``argv`` argument the + same way the regular Python parses command line arguments: see + :ref:`Command Line Arguments `. + + .. c:member:: int use_environment + + See :c:member:`PyConfig.use_environment`. + + .. c:member:: int utf8_mode + + If non-zero, enable the UTF-8 mode. + +Preinitialization with PyPreConfig +---------------------------------- + +Functions to preinitialize Python: + +.. c:function:: PyStatus Py_PreInitialize(const PyPreConfig *preconfig) + + Preinitialize Python from *preconfig* preconfiguration. + +.. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) + + Preinitialize Python from *preconfig* preconfiguration and command line + arguments (bytes strings). + +.. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) + + Preinitialize Python from *preconfig* preconfiguration and command line + arguments (wide strings). + +The caller is responsible to handle exceptions (error or exit) using +:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. + +For :ref:`Python Configuration ` +(:c:func:`PyPreConfig_InitPythonConfig`), if Python is initialized with +command line arguments, the command line arguments must also be passed to +preinitialize Python, since they have an effect on the pre-configuration +like encodings. For example, the :option:`-X utf8 <-X>` command line option +enables the UTF-8 Mode. + +``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and +before :c:func:`Py_InitializeFromConfig` to install a custom memory allocator. +It can be called before :c:func:`Py_PreInitialize` if +:c:member:`PyPreConfig.allocator` is set to ``PYMEM_ALLOCATOR_NOT_SET``. + +Python memory allocation functions like :c:func:`PyMem_RawMalloc` must not be +used before Python preinitialization, whereas calling directly ``malloc()`` and +``free()`` is always safe. :c:func:`Py_DecodeLocale` must not be called before +the preinitialization. + +Example using the preinitialization to enable the UTF-8 Mode:: + + PyStatus status; + PyPreConfig preconfig; + PyPreConfig_InitPythonConfig(&preconfig); + + preconfig.utf8_mode = 1; + + status = Py_PreInitialize(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + /* at this point, Python will speak UTF-8 */ + + Py_Initialize(); + /* ... use Python API here ... */ + Py_Finalize(); + + +PyConfig +-------- + +.. c:type:: PyConfig + + Structure containing most parameters to configure Python. + + Structure methods: + + .. c:function:: void PyConfig_InitPythonConfig(PyConfig *config) + + Initialize configuration with :ref:`Python Configuration + `. + + .. c:function:: void PyConfig_InitIsolatedConfig(PyConfig *config) + + Initialize configuration with :ref:`Isolated Configuration + `. + + .. c:function:: PyStatus PyConfig_SetString(PyConfig *config, wchar_t * const *config_str, const wchar_t *str) + + Copy the wide character string *str* into ``*config_str``. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetBytesString(PyConfig *config, wchar_t * const *config_str, const char *str) + + Decode *str* using ``Py_DecodeLocale()`` and set the result into ``*config_str``. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetArgv(PyConfig *config, int argc, wchar_t * const *argv) + + Set command line arguments from wide character strings. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetBytesArgv(PyConfig *config, int argc, char * const *argv) + + Set command line arguments: decode bytes using :c:func:`Py_DecodeLocale`. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items) + + Set the list of wide strings *list* to *length* and *items*. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_Read(PyConfig *config) + + Read all Python configuration. + + Fields which are already initialized are left unchanged. + + Preinitialize Python if needed. + + .. c:function:: void PyConfig_Clear(PyConfig *config) + + Release configuration memory. + + Most ``PyConfig`` methods preinitialize Python if needed. In that case, the + Python preinitialization configuration in based on the :c:type:`PyConfig`. + If configuration fields which are in common with :c:type:`PyPreConfig` are + tuned, they must be set before calling a :c:type:`PyConfig` method: + + * :c:member:`~PyConfig.dev_mode` + * :c:member:`~PyConfig.isolated` + * :c:member:`~PyConfig.parse_argv` + * :c:member:`~PyConfig.use_environment` + + Moreover, if :c:func:`PyConfig_SetArgv` or :c:func:`PyConfig_SetBytesArgv` + is used, this method must be called first, before other methods, since the + preinitialization configuration depends on command line arguments (if + :c:member:`parse_argv` is non-zero). + + The caller of these methods is responsible to handle exceptions (error or + exit) using ``PyStatus_Exception()`` and ``Py_ExitStatusException()``. + + Structure fields: + + .. c:member:: PyWideStringList argv + + Command line arguments, :data:`sys.argv`. See + :c:member:`~PyConfig.parse_argv` to parse :c:member:`~PyConfig.argv` the + same way the regular Python parses Python command line arguments. If + :c:member:`~PyConfig.argv` is empty, an empty string is added to ensure + that :data:`sys.argv` always exists and is never empty. + + See also the :c:member:`~PyConfig.orig_argv` member. + + .. c:member:: wchar_t* base_exec_prefix + + :data:`sys.base_exec_prefix`. + + .. c:member:: wchar_t* base_executable + + :data:`sys._base_executable`: ``__PYVENV_LAUNCHER__`` environment + variable value, or copy of :c:member:`PyConfig.executable`. + + .. c:member:: wchar_t* base_prefix + + :data:`sys.base_prefix`. + + .. c:member:: wchar_t* platlibdir + + :data:`sys.platlibdir`: platform library directory name, set at configure time + by ``--with-platlibdir``, overrideable by the ``PYTHONPLATLIBDIR`` + environment variable. + + .. versionadded:: 3.9 + + .. c:member:: int buffered_stdio + + If equals to 0, enable unbuffered mode, making the stdout and stderr + streams unbuffered. + + stdin is always opened in buffered mode. + + .. c:member:: int bytes_warning + + If equals to 1, issue a warning when comparing :class:`bytes` or + :class:`bytearray` with :class:`str`, or comparing :class:`bytes` with + :class:`int`. If equal or greater to 2, raise a :exc:`BytesWarning` + exception. + + .. c:member:: wchar_t* check_hash_pycs_mode + + Control the validation behavior of hash-based ``.pyc`` files (see + :pep:`552`): :option:`--check-hash-based-pycs` command line option value. + + Valid values: ``always``, ``never`` and ``default``. + + The default value is: ``default``. + + .. c:member:: int configure_c_stdio + + If non-zero, configure C standard streams (``stdio``, ``stdout``, + ``stdout``). For example, set their mode to ``O_BINARY`` on Windows. + + .. c:member:: int dev_mode + + If non-zero, enable the :ref:`Python Development Mode `. + + .. c:member:: int dump_refs + + If non-zero, dump all objects which are still alive at exit. + + ``Py_TRACE_REFS`` macro must be defined in build. + + .. c:member:: wchar_t* exec_prefix + + :data:`sys.exec_prefix`. + + .. c:member:: wchar_t* executable + + :data:`sys.executable`. + + .. c:member:: int faulthandler + + If non-zero, call :func:`faulthandler.enable` at startup. + + .. c:member:: wchar_t* filesystem_encoding + + Filesystem encoding, :func:`sys.getfilesystemencoding`. + + .. c:member:: wchar_t* filesystem_errors + + Filesystem encoding errors, :func:`sys.getfilesystemencodeerrors`. + + .. c:member:: unsigned long hash_seed + .. c:member:: int use_hash_seed + + Randomized hash function seed. + + If :c:member:`~PyConfig.use_hash_seed` is zero, a seed is chosen randomly + at Pythonstartup, and :c:member:`~PyConfig.hash_seed` is ignored. + + .. c:member:: wchar_t* home + + Python home directory. + + Initialized from :envvar:`PYTHONHOME` environment variable value by + default. + + .. c:member:: int import_time + + If non-zero, profile import time. + + .. c:member:: int inspect + + Enter interactive mode after executing a script or a command. + + .. c:member:: int install_signal_handlers + + Install signal handlers? + + .. c:member:: int interactive + + Interactive mode. + + .. c:member:: int isolated + + If greater than 0, enable isolated mode: + + * :data:`sys.path` contains neither the script's directory (computed from + ``argv[0]`` or the current directory) nor the user's site-packages + directory. + * Python REPL doesn't import :mod:`readline` nor enable default readline + configuration on interactive prompts. + * Set :c:member:`~PyConfig.use_environment` and + :c:member:`~PyConfig.user_site_directory` to 0. + + .. c:member:: int legacy_windows_stdio + + If non-zero, use :class:`io.FileIO` instead of + :class:`io.WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout` + and :data:`sys.stderr`. + + Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for + Windows specific code. + + .. c:member:: int malloc_stats + + If non-zero, dump statistics on :ref:`Python pymalloc memory allocator + ` at exit. + + The option is ignored if Python is built using ``--without-pymalloc``. + + .. c:member:: wchar_t* pythonpath_env + + Module search paths as a string separated by ``DELIM`` + (:data:`os.path.pathsep`). + + Initialized from :envvar:`PYTHONPATH` environment variable value by + default. + + .. c:member:: PyWideStringList module_search_paths + .. c:member:: int module_search_paths_set + + :data:`sys.path`. If :c:member:`~PyConfig.module_search_paths_set` is + equal to 0, the :c:member:`~PyConfig.module_search_paths` is overridden + by the function calculating the :ref:`Path Configuration + `. + + .. c:member:: int optimization_level + + Compilation optimization level: + + * 0: Peephole optimizer (and ``__debug__`` is set to ``True``) + * 1: Remove assertions, set ``__debug__`` to ``False`` + * 2: Strip docstrings + + .. c:member:: PyWideStringList orig_argv + + The list of the original command line arguments passed to the Python + executable. + + If :c:member:`~PyConfig.orig_argv` list is empty and + :c:member:`~PyConfig.argv` is not a list only containing an empty + string, :c:func:`PyConfig_Read()` copies :c:member:`~PyConfig.argv` into + :c:member:`~PyConfig.orig_argv` before modifying + :c:member:`~PyConfig.argv` (if :c:member:`~PyConfig.parse_argv` is + non-zero). + + See also the :c:member:`~PyConfig.argv` member and the + :c:func:`Py_GetArgcArgv` function. + + .. versionadded:: 3.10 + + .. c:member:: int parse_argv + + If non-zero, parse :c:member:`~PyConfig.argv` the same way the regular + Python command line arguments, and strip Python arguments from + :c:member:`~PyConfig.argv`: see :ref:`Command Line Arguments + `. + + .. c:member:: int parser_debug + + If non-zero, turn on parser debugging output (for expert only, depending + on compilation options). + + .. c:member:: int pathconfig_warnings + + If equal to 0, suppress warnings when calculating the :ref:`Path + Configuration ` (Unix only, Windows does not log any + warning). Otherwise, warnings are written into ``stderr``. + + .. c:member:: wchar_t* prefix + + :data:`sys.prefix`. + + .. c:member:: wchar_t* program_name + + Program name. Used to initialize :c:member:`~PyConfig.executable`, and in + early error messages. + + .. c:member:: wchar_t* pycache_prefix + + :data:`sys.pycache_prefix`: ``.pyc`` cache prefix. + + If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``. + + .. c:member:: int quiet + + Quiet mode. For example, don't display the copyright and version messages + in interactive mode. + + .. c:member:: wchar_t* run_command + + ``python3 -c COMMAND`` argument. Used by :c:func:`Py_RunMain`. + + .. c:member:: wchar_t* run_filename + + ``python3 FILENAME`` argument. Used by :c:func:`Py_RunMain`. + + .. c:member:: wchar_t* run_module + + ``python3 -m MODULE`` argument. Used by :c:func:`Py_RunMain`. + + .. c:member:: int show_ref_count + + Show total reference count at exit? + + Set to 1 by :option:`-X showrefcount <-X>` command line option. + + Need a debug build of Python (``Py_REF_DEBUG`` macro must be defined). + + .. c:member:: int site_import + + Import the :mod:`site` module at startup? + + .. c:member:: int skip_source_first_line + + Skip the first line of the source? + + .. c:member:: wchar_t* stdio_encoding + .. c:member:: wchar_t* stdio_errors + + Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and + :data:`sys.stderr`. + + .. c:member:: int tracemalloc + + If non-zero, call :func:`tracemalloc.start` at startup. + + .. c:member:: int use_environment + + If greater than 0, use :ref:`environment variables `. + + .. c:member:: int user_site_directory + + If non-zero, add user site directory to :data:`sys.path`. + + .. c:member:: int verbose + + If non-zero, enable verbose mode. + + .. c:member:: PyWideStringList warnoptions + + :data:`sys.warnoptions`: options of the :mod:`warnings` module to build + warnings filters: lowest to highest priority. + + The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse + order: the last :c:member:`PyConfig.warnoptions` item becomes the first + item of :data:`warnings.filters` which is checked first (highest + priority). + + .. c:member:: int write_bytecode + + If non-zero, write ``.pyc`` files. + + :data:`sys.dont_write_bytecode` is initialized to the inverted value of + :c:member:`~PyConfig.write_bytecode`. + + .. c:member:: PyWideStringList xoptions + + :data:`sys._xoptions`. + +If ``parse_argv`` is non-zero, ``argv`` arguments are parsed the same +way the regular Python parses command line arguments, and Python +arguments are stripped from ``argv``: see :ref:`Command Line Arguments +`. + +The ``xoptions`` options are parsed to set other options: see :option:`-X` +option. + +.. versionchanged:: 3.9 + + The ``show_alloc_count`` field has been removed. + + +Initialization with PyConfig +---------------------------- + +Function to initialize Python: + +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration. + +The caller is responsible to handle exceptions (error or exit) using +:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. + +If ``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or +``PyImport_ExtendInittab()`` are used, they must be set or called after Python +preinitialization and before the Python initialization. + +Example setting the program name:: + + void init_python(void) + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + + /* Set the program name. Implicitly preinitialize Python. */ + status = PyConfig_SetString(&config, &config.program_name, + L"/path/to/my_program"); + if (PyStatus_Exception(status)) { + goto fail; + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + PyConfig_Clear(&config); + return; + + fail: + PyConfig_Clear(&config); + Py_ExitStatusException(status); + } + +More complete example modifying the default configuration, read the +configuration, and then override some parameters:: + + PyStatus init_python(const char *program_name) + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + + /* Set the program name before reading the configuration + (decode byte string from the locale encoding). + + Implicitly preinitialize Python. */ + status = PyConfig_SetBytesString(&config, &config.program_name, + program_name); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Read all configuration at once */ + status = PyConfig_Read(&config); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Append our custom search path to sys.path */ + status = PyWideStringList_Append(&config.module_search_paths, + L"/path/to/more/modules"); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Override executable computed by PyConfig_Read() */ + status = PyConfig_SetString(&config, &config.executable, + L"/path/to/my_executable"); + if (PyStatus_Exception(status)) { + goto done; + } + + status = Py_InitializeFromConfig(&config); + + done: + PyConfig_Clear(&config); + return status; + } + + +.. _init-isolated-conf: + +Isolated Configuration +---------------------- + +:c:func:`PyPreConfig_InitIsolatedConfig` and +:c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to +isolate Python from the system. For example, to embed Python into an +application. + +This configuration ignores global configuration variables, environments +variables, command line arguments (:c:member:`PyConfig.argv` is not parsed) +and user site directory. The C standard streams (ex: ``stdout``) and the +LC_CTYPE locale are left unchanged. Signal handlers are not installed. + +Configuration files are still used with this configuration. Set the +:ref:`Path Configuration ` ("output fields") to ignore these +configuration files and avoid the function computing the default path +configuration. + + +.. _init-python-config: + +Python Configuration +-------------------- + +:c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` +functions create a configuration to build a customized Python which behaves as +the regular Python. + +Environments variables and command line arguments are used to configure +Python, whereas global configuration variables are ignored. + +This function enables C locale coercion (:pep:`538`) and UTF-8 Mode +(:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and +:envvar:`PYTHONCOERCECLOCALE` environment variables. + +Example of customized Python always running in isolated mode:: + + int main(int argc, char **argv) + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + + /* Decode command line arguments. + Implicitly preinitialize Python (in isolated mode). */ + status = PyConfig_SetBytesArgv(&config, argc, argv); + if (PyStatus_Exception(status)) { + goto fail; + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + PyConfig_Clear(&config); + + return Py_RunMain(); + + fail: + PyConfig_Clear(&config); + if (PyStatus_IsExit(status)) { + return status.exitcode; + } + /* Display the error message and exit the process with + non-zero exit code */ + Py_ExitStatusException(status); + } + + +.. _init-path-config: + +Path Configuration +------------------ + +:c:type:`PyConfig` contains multiple fields for the path configuration: + +* Path configuration inputs: + + * :c:member:`PyConfig.home` + * :c:member:`PyConfig.platlibdir` + * :c:member:`PyConfig.pathconfig_warnings` + * :c:member:`PyConfig.program_name` + * :c:member:`PyConfig.pythonpath_env` + * current working directory: to get absolute paths + * ``PATH`` environment variable to get the program full path + (from :c:member:`PyConfig.program_name`) + * ``__PYVENV_LAUNCHER__`` environment variable + * (Windows only) Application paths in the registry under + "Software\Python\PythonCore\X.Y\PythonPath" of HKEY_CURRENT_USER and + HKEY_LOCAL_MACHINE (where X.Y is the Python version). + +* Path configuration output fields: + + * :c:member:`PyConfig.base_exec_prefix` + * :c:member:`PyConfig.base_executable` + * :c:member:`PyConfig.base_prefix` + * :c:member:`PyConfig.exec_prefix` + * :c:member:`PyConfig.executable` + * :c:member:`PyConfig.module_search_paths_set`, + :c:member:`PyConfig.module_search_paths` + * :c:member:`PyConfig.prefix` + +If at least one "output field" is not set, Python calculates the path +configuration to fill unset fields. If +:c:member:`~PyConfig.module_search_paths_set` is equal to 0, +:c:member:`~PyConfig.module_search_paths` is overridden and +:c:member:`~PyConfig.module_search_paths_set` is set to 1. + +It is possible to completely ignore the function calculating the default +path configuration by setting explicitly all path configuration output +fields listed above. A string is considered as set even if it is non-empty. +``module_search_paths`` is considered as set if +``module_search_paths_set`` is set to 1. In this case, path +configuration input fields are ignored as well. + +Set :c:member:`~PyConfig.pathconfig_warnings` to 0 to suppress warnings when +calculating the path configuration (Unix only, Windows does not log any warning). + +If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix` +fields are not set, they inherit their value from :c:member:`~PyConfig.prefix` +and :c:member:`~PyConfig.exec_prefix` respectively. + +:c:func:`Py_RunMain` and :c:func:`Py_Main` modify :data:`sys.path`: + +* If :c:member:`~PyConfig.run_filename` is set and is a directory which contains a + ``__main__.py`` script, prepend :c:member:`~PyConfig.run_filename` to + :data:`sys.path`. +* If :c:member:`~PyConfig.isolated` is zero: + + * If :c:member:`~PyConfig.run_module` is set, prepend the current directory + to :data:`sys.path`. Do nothing if the current directory cannot be read. + * If :c:member:`~PyConfig.run_filename` is set, prepend the directory of the + filename to :data:`sys.path`. + * Otherwise, prepend an empty string to :data:`sys.path`. + +If :c:member:`~PyConfig.site_import` is non-zero, :data:`sys.path` can be +modified by the :mod:`site` module. If +:c:member:`~PyConfig.user_site_directory` is non-zero and the user's +site-package directory exists, the :mod:`site` module appends the user's +site-package directory to :data:`sys.path`. + +The following configuration files are used by the path configuration: + +* ``pyvenv.cfg`` +* ``python._pth`` (Windows only) +* ``pybuilddir.txt`` (Unix only) + +The ``__PYVENV_LAUNCHER__`` environment variable is used to set +:c:member:`PyConfig.base_executable` + + +Py_RunMain() +------------ + +.. c:function:: int Py_RunMain(void) + + Execute the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. + + By default and when if :option:`-i` option is used, run the REPL. + + Finally, finalizes Python and returns an exit status that can be passed to + the ``exit()`` function. + +See :ref:`Python Configuration ` for an example of +customized Python always running in isolated mode using +:c:func:`Py_RunMain`. + + +Py_GetArgcArgv() +---------------- + +.. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv) + + Get the original command line arguments, before Python modified them. + + See also :c:member:`PyConfig.orig_argv` member. + + +Multi-Phase Initialization Private Provisional API +-------------------------------------------------- + +This section is a private provisional API introducing multi-phase +initialization, the core feature of the :pep:`432`: + +* "Core" initialization phase, "bare minimum Python": + + * Builtin types; + * Builtin exceptions; + * Builtin and frozen modules; + * The :mod:`sys` module is only partially initialized + (ex: :data:`sys.path` doesn't exist yet). + +* "Main" initialization phase, Python is fully initialized: + + * Install and configure :mod:`importlib`; + * Apply the :ref:`Path Configuration `; + * Install signal handlers; + * Finish :mod:`sys` module initialization (ex: create :data:`sys.stdout` + and :data:`sys.path`); + * Enable optional features like :mod:`faulthandler` and :mod:`tracemalloc`; + * Import the :mod:`site` module; + * etc. + +Private provisional API: + +* :c:member:`PyConfig._init_main`: if set to 0, + :c:func:`Py_InitializeFromConfig` stops at the "Core" initialization phase. +* :c:member:`PyConfig._isolated_interpreter`: if non-zero, + disallow threads, subprocesses and fork. + +.. c:function:: PyStatus _Py_InitializeMain(void) + + Move to the "Main" initialization phase, finish the Python initialization. + +No module is imported during the "Core" phase and the ``importlib`` module is +not configured: the :ref:`Path Configuration ` is only +applied during the "Main" phase. It may allow to customize Python in Python to +override or tune the :ref:`Path Configuration `, maybe +install a custom :data:`sys.meta_path` importer or an import hook, etc. + +It may become possible to calculatin the :ref:`Path Configuration +` in Python, after the Core phase and before the Main phase, +which is one of the :pep:`432` motivation. + +The "Core" phase is not properly defined: what should be and what should +not be available at this phase is not specified yet. The API is marked +as private and provisional: the API can be modified or even be removed +anytime until a proper public API is designed. + +Example running Python code between "Core" and "Main" initialization +phases:: + + void init_python(void) + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + config._init_main = 0; + + /* ... customize 'config' configuration ... */ + + status = Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + /* Use sys.stderr because sys.stdout is only created + by _Py_InitializeMain() */ + int res = PyRun_SimpleString( + "import sys; " + "print('Run Python code before _Py_InitializeMain', " + "file=sys.stderr)"); + if (res < 0) { + exit(1); + } + + /* ... put more configuration code here ... */ + + status = _Py_InitializeMain(); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + } diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 69aef0da04f3ba..7ca8693afab79c 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _api-intro: @@ -69,10 +69,12 @@ standard headers) have one of the prefixes ``Py`` or ``_Py``. Names beginning with ``_Py`` are for internal use by the Python implementation and should not be used by extension writers. Structure member names do not have a reserved prefix. -**Important:** user code should never define names that begin with ``Py`` or -``_Py``. This confuses the reader, and jeopardizes the portability of the user -code to future Python versions, which may define additional names beginning with -one of these prefixes. +.. note:: + + User code should never define names that begin with ``Py`` or ``_Py``. This + confuses the reader, and jeopardizes the portability of the user code to + future Python versions, which may define additional names beginning with one + of these prefixes. The header files are typically installed with Python. On Unix, these are located in the directories :file:`{prefix}/include/pythonversion/` and @@ -90,9 +92,9 @@ multi-platform builds since the platform independent headers under :envvar:`prefix` include the platform specific headers from :envvar:`exec_prefix`. -C++ users should note that though the API is defined entirely using C, the -header files do properly declare the entry points to be ``extern "C"``, so there -is no need to do anything special to use the API from C++. +C++ users should note that although the API is defined entirely using C, the +header files properly declare the entry points to be ``extern "C"``. As a result, +there is no need to do anything special to use the API from C++. Useful macros @@ -105,11 +107,24 @@ complete listing. .. c:macro:: Py_UNREACHABLE() - Use this when you have a code path that you do not expect to be reached. + Use this when you have a code path that cannot be reached by design. For example, in the ``default:`` clause in a ``switch`` statement for which all possible values are covered in ``case`` statements. Use this in places where you might be tempted to put an ``assert(0)`` or ``abort()`` call. + In release mode, the macro helps the compiler to optimize the code, and + avoids a warning about unreachable code. For example, the macro is + implemented with ``__builtin_unreachable()`` on GCC in release mode. + + A use for ``Py_UNREACHABLE()`` is following a call a function that + never returns but that is not declared :c:macro:`_Py_NO_RETURN`. + + If a code path is very unlikely code but can be reached under exceptional + case, this macro must not be used. For example, under low memory condition + or if a system call returns a value out of the expected range. In this + case, it's better to report the error to the caller. If the error cannot + be reported to caller, :c:func:`Py_FatalError` can be used. + .. versionadded:: 3.7 .. c:macro:: Py_ABS(x) @@ -150,16 +165,61 @@ complete listing. .. c:macro:: Py_GETENV(s) - Like ``getenv(s)``, but returns *NULL* if :option:`-E` was passed on the + Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the command line (i.e. if ``Py_IgnoreEnvironmentFlag`` is set). .. c:macro:: Py_UNUSED(arg) Use this for unused arguments in a function definition to silence compiler - warnings, e.g. ``PyObject* func(PyObject *Py_UNUSED(ignored))``. + warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. .. versionadded:: 3.4 +.. c:macro:: Py_DEPRECATED(version) + + Use this for deprecated declarations. The macro must be placed before the + symbol name. + + Example:: + + Py_DEPRECATED(3.8) PyAPI_FUNC(int) Py_OldFunction(void); + + .. versionchanged:: 3.8 + MSVC support was added. + +.. c:macro:: PyDoc_STRVAR(name, str) + + Creates a variable with name ``name`` that can be used in docstrings. + If Python is built without docstrings, the value will be empty. + + Use :c:macro:`PyDoc_STRVAR` for docstrings to support building + Python without docstrings, as specified in :pep:`7`. + + Example:: + + PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); + + static PyMethodDef deque_methods[] = { + // ... + {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, + // ... + } + +.. c:macro:: PyDoc_STR(str) + + Creates a docstring for the given input string or an empty string + if docstrings are disabled. + + Use :c:macro:`PyDoc_STR` in specifying docstrings to support + building Python without docstrings, as specified in :pep:`7`. + + Example:: + + static PyMethodDef pysqlite_row_methods[] = { + {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, + PyDoc_STR("Returns the keys of the row.")}, + {NULL, NULL} + }; .. _api-objects: @@ -169,13 +229,13 @@ Objects, Types and Reference Counts .. index:: object: type Most Python/C API functions have one or more arguments as well as a return value -of type :c:type:`PyObject\*`. This type is a pointer to an opaque data type +of type :c:type:`PyObject*`. This type is a pointer to an opaque data type representing an arbitrary Python object. Since all Python object types are treated the same way by the Python language in most situations (e.g., assignments, scope rules, and argument passing), it is only fitting that they should be represented by a single C type. Almost all Python objects live on the heap: you never declare an automatic or static variable of type -:c:type:`PyObject`, only pointer variables of type :c:type:`PyObject\*` can be +:c:type:`PyObject`, only pointer variables of type :c:type:`PyObject*` can be declared. The sole exception are the type objects; since these must never be deallocated, they are typically static :c:type:`PyTypeObject` objects. @@ -436,7 +496,7 @@ Types There are few other data types that play a significant role in the Python/C API; most are simple C types such as :c:type:`int`, :c:type:`long`, -:c:type:`double` and :c:type:`char\*`. A few structure types are used to +:c:type:`double` and :c:type:`char*`. A few structure types are used to describe static tables used to list the functions exported by a module or the data attributes of a new object type, and another is used to describe the value of a complex number. These will be discussed together with the functions that @@ -461,7 +521,7 @@ functions in the Python/C API can raise exceptions, unless an explicit claim is made otherwise in a function's documentation. In general, when a function encounters an error, it sets an exception, discards any object references that it owns, and returns an error indicator. If not documented otherwise, this -indicator is either *NULL* or ``-1``, depending on the function's return type. +indicator is either ``NULL`` or ``-1``, depending on the function's return type. A few functions return a Boolean true/false result, with false indicating an error. Very few functions return no explicit error indicator or have an ambiguous return value, and require explicit testing for errors with @@ -476,13 +536,13 @@ using global storage in an unthreaded application). A thread can be in one of two states: an exception has occurred, or not. The function :c:func:`PyErr_Occurred` can be used to check for this: it returns a borrowed reference to the exception type object when an exception has occurred, and -*NULL* otherwise. There are a number of functions to set the exception state: +``NULL`` otherwise. There are a number of functions to set the exception state: :c:func:`PyErr_SetString` is the most common (though not the most general) function to set the exception state, and :c:func:`PyErr_Clear` clears the exception state. The full exception state consists of three objects (all of which can be -*NULL*): the exception type, the corresponding exception value, and the +``NULL``): the exception type, the corresponding exception value, and the traceback. These have the same meanings as the Python result of ``sys.exc_info()``; however, they are not the same: the Python objects represent the last exception being handled by a Python :keyword:`try` ... @@ -583,10 +643,10 @@ Here is the corresponding C code, in all its glory:: This example represents an endorsed use of the ``goto`` statement in C! It illustrates the use of :c:func:`PyErr_ExceptionMatches` and :c:func:`PyErr_Clear` to handle specific exceptions, and the use of -:c:func:`Py_XDECREF` to dispose of owned references that may be *NULL* (note the +:c:func:`Py_XDECREF` to dispose of owned references that may be ``NULL`` (note the ``'X'`` in the name; :c:func:`Py_DECREF` would crash when confronted with a -*NULL* reference). It is important that the variables used to hold owned -references are initialized to *NULL* for this to work; likewise, the proposed +``NULL`` reference). It is important that the variables used to hold owned +references are initialized to ``NULL`` for this to work; likewise, the proposed return value is initialized to ``-1`` (failure) and only set to success after the final call made is successful. diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 2ba444d1de6d24..a2992b3452f91c 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _iterator: @@ -16,8 +16,8 @@ There are two functions specifically for working with iterators. Return the next value from the iteration *o*. The object must be an iterator (it is up to the caller to check this). If there are no remaining values, - returns *NULL* with no exception set. If an error occurs while retrieving - the item, returns *NULL* and passes along the exception. + returns ``NULL`` with no exception set. If an error occurs while retrieving + the item, returns ``NULL`` and passes along the exception. To write a loop which iterates over an iterator, the C code should look something like this:: @@ -29,7 +29,7 @@ something like this:: /* propagate error */ } - while (item = PyIter_Next(iterator)) { + while ((item = PyIter_Next(iterator))) { /* do something with item */ ... /* release reference when done */ diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 82cb4eba8ab743..4d91e4a224fe82 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _iterator-objects: diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 5b263a7b1cdf04..0bc0785f200d4c 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _listobjects: @@ -33,7 +33,7 @@ List Objects .. c:function:: PyObject* PyList_New(Py_ssize_t len) - Return a new list of length *len* on success, or *NULL* on failure. + Return a new list of length *len* on success, or ``NULL`` on failure. .. note:: @@ -59,9 +59,9 @@ List Objects .. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) Return the object at position *index* in the list pointed to by *list*. The - position must be positive, indexing from the end of the list is not - supported. If *index* is out of bounds, return *NULL* and set an - :exc:`IndexError` exception. + position must be non-negative; indexing from the end of the list is not + supported. If *index* is out of bounds (<0 or >=len(list)), + return ``NULL`` and set an :exc:`IndexError` exception. .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) @@ -71,8 +71,9 @@ List Objects .. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item) - Set the item at index *index* in list to *item*. Return ``0`` on success - or ``-1`` on failure. + Set the item at index *index* in list to *item*. Return ``0`` on success. + If *index* is out of bounds, return ``-1`` and set an :exc:`IndexError` + exception. .. note:: @@ -110,18 +111,17 @@ List Objects .. c:function:: PyObject* PyList_GetSlice(PyObject *list, Py_ssize_t low, Py_ssize_t high) Return a list of the objects in *list* containing the objects *between* *low* - and *high*. Return *NULL* and set an exception if unsuccessful. Analogous - to ``list[low:high]``. Negative indices, as when slicing from Python, are not - supported. + and *high*. Return ``NULL`` and set an exception if unsuccessful. Analogous + to ``list[low:high]``. Indexing from the end of the list is not supported. .. c:function:: int PyList_SetSlice(PyObject *list, Py_ssize_t low, Py_ssize_t high, PyObject *itemlist) Set the slice of *list* between *low* and *high* to the contents of *itemlist*. Analogous to ``list[low:high] = itemlist``. The *itemlist* may - be *NULL*, indicating the assignment of an empty list (slice deletion). - Return ``0`` on success, ``-1`` on failure. Negative indices, as when - slicing from Python, are not supported. + be ``NULL``, indicating the assignment of an empty list (slice deletion). + Return ``0`` on success, ``-1`` on failure. Indexing from the end of the + list is not supported. .. c:function:: int PyList_Sort(PyObject *list) @@ -142,10 +142,3 @@ List Objects Return a new tuple object containing the contents of *list*; equivalent to ``tuple(list)``. - - -.. c:function:: int PyList_ClearFreeList() - - Clear the free list. Return the total number of freed items. - - .. versionadded:: 3.3 diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 8093f4b627a2b9..4d859fb641df3d 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _longobjects: @@ -38,55 +38,53 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: PyObject* PyLong_FromLong(long v) - Return a new :c:type:`PyLongObject` object from *v*, or *NULL* on failure. + Return a new :c:type:`PyLongObject` object from *v*, or ``NULL`` on failure. The current implementation keeps an array of integer objects for all integers between ``-5`` and ``256``, when you create an int in that range you actually - just get back a reference to the existing object. So it should be possible to - change the value of ``1``. I suspect the behaviour of Python in this case is - undefined. :-) + just get back a reference to the existing object. .. c:function:: PyObject* PyLong_FromUnsignedLong(unsigned long v) Return a new :c:type:`PyLongObject` object from a C :c:type:`unsigned long`, or - *NULL* on failure. + ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromSsize_t(Py_ssize_t v) Return a new :c:type:`PyLongObject` object from a C :c:type:`Py_ssize_t`, or - *NULL* on failure. + ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromSize_t(size_t v) Return a new :c:type:`PyLongObject` object from a C :c:type:`size_t`, or - *NULL* on failure. + ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromLongLong(long long v) - Return a new :c:type:`PyLongObject` object from a C :c:type:`long long`, or *NULL* + Return a new :c:type:`PyLongObject` object from a C :c:type:`long long`, or ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) Return a new :c:type:`PyLongObject` object from a C :c:type:`unsigned long long`, - or *NULL* on failure. + or ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromDouble(double v) Return a new :c:type:`PyLongObject` object from the integer part of *v*, or - *NULL* on failure. + ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromString(const char *str, char **pend, int base) Return a new :c:type:`PyLongObject` based on the string value in *str*, which - is interpreted according to the radix in *base*. If *pend* is non-*NULL*, + is interpreted according to the radix in *base*. If *pend* is non-``NULL``, *\*pend* will point to the first character in *str* which follows the representation of the number. If *base* is ``0``, *str* is interpreted using the :ref:`integers` definition; in this case, leading zeros in a @@ -96,23 +94,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. are no digits, :exc:`ValueError` will be raised. -.. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) - - Convert a sequence of Unicode digits to a Python integer value. The Unicode - string is first encoded to a byte string using :c:func:`PyUnicode_EncodeDecimal` - and then converted using :c:func:`PyLong_FromString`. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyLong_FromUnicodeObject`. - - .. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) Convert a sequence of Unicode digits in the string *u* to a Python integer - value. The Unicode string is first encoded to a byte string using - :c:func:`PyUnicode_EncodeDecimal` and then converted using - :c:func:`PyLong_FromString`. + value. .. versionadded:: 3.3 @@ -131,9 +116,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. single: OverflowError (built-in exception) Return a C :c:type:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` or - :meth:`__int__` method (if present) to convert it to a - :c:type:`PyLongObject`. + instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method + (if present) to convert it to a :c:type:`PyLongObject`. Raise :exc:`OverflowError` if the value of *obj* is out of range for a :c:type:`long`. @@ -143,16 +127,15 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. - .. deprecated:: 3.8 - Using :meth:`__int__` is deprecated. + .. versionchanged:: 3.10 + This function will no longer use :meth:`__int__`. .. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow) Return a C :c:type:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` or - :meth:`__int__` method (if present) to convert it to a - :c:type:`PyLongObject`. + instance of :c:type:`PyLongObject`, first call its :meth:`__index__` + method (if present) to convert it to a :c:type:`PyLongObject`. If the value of *obj* is greater than :const:`LONG_MAX` or less than :const:`LONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, and @@ -164,8 +147,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. - .. deprecated:: 3.8 - Using :meth:`__int__` is deprecated. + .. versionchanged:: 3.10 + This function will no longer use :meth:`__int__`. .. c:function:: long long PyLong_AsLongLong(PyObject *obj) @@ -174,31 +157,29 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. single: OverflowError (built-in exception) Return a C :c:type:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` or - :meth:`__int__` method (if present) to convert it to a - :c:type:`PyLongObject`. + instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method + (if present) to convert it to a :c:type:`PyLongObject`. Raise :exc:`OverflowError` if the value of *obj* is out of range for a - :c:type:`long`. + :c:type:`long long`. Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. - .. deprecated:: 3.8 - Using :meth:`__int__` is deprecated. + .. versionchanged:: 3.10 + This function will no longer use :meth:`__int__`. .. c:function:: long long PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow) Return a C :c:type:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__index__` or - :meth:`__int__` method (if present) to convert it to a - :c:type:`PyLongObject`. + instance of :c:type:`PyLongObject`, first call its :meth:`__index__` method + (if present) to convert it to a :c:type:`PyLongObject`. - If the value of *obj* is greater than :const:`PY_LLONG_MAX` or less than - :const:`PY_LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, + If the value of *obj* is greater than :const:`LLONG_MAX` or less than + :const:`LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual. @@ -209,8 +190,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. - .. deprecated:: 3.8 - Using :meth:`__int__` is deprecated. + .. versionchanged:: 3.10 + This function will no longer use :meth:`__int__`. .. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong) @@ -280,40 +261,41 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: unsigned long PyLong_AsUnsignedLongMask(PyObject *obj) - Return a C :c:type:`unsigned long` representation of *obj*. If *obj* - is not an instance of :c:type:`PyLongObject`, first call its - :meth:`__index__` or :meth:`__int__` method (if present) to convert - it to a :c:type:`PyLongObject`. + Return a C :c:type:`unsigned long` representation of *obj*. If *obj* is not + an instance of :c:type:`PyLongObject`, first call its :meth:`__index__` + method (if present) to convert it to a :c:type:`PyLongObject`. If the value of *obj* is out of range for an :c:type:`unsigned long`, return the reduction of that value modulo ``ULONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long)-1`` on error. Use :c:func:`PyErr_Occurred` to + disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. - .. deprecated:: 3.8 - Using :meth:`__int__` is deprecated. + .. versionchanged:: 3.10 + This function will no longer use :meth:`__int__`. .. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj) Return a C :c:type:`unsigned long long` representation of *obj*. If *obj* is not an instance of :c:type:`PyLongObject`, first call its - :meth:`__index__` or :meth:`__int__` method (if present) to convert - it to a :c:type:`PyLongObject`. + :meth:`__index__` method (if present) to convert it to a + :c:type:`PyLongObject`. If the value of *obj* is out of range for an :c:type:`unsigned long long`, - return the reduction of that value modulo ``PY_ULLONG_MAX + 1``. + return the reduction of that value modulo ``ULLONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long long)-1`` on error. Use :c:func:`PyErr_Occurred` + to disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. - .. deprecated:: 3.8 - Using :meth:`__int__` is deprecated. + .. versionchanged:: 3.10 + This function will no longer use :meth:`__int__`. .. c:function:: double PyLong_AsDouble(PyObject *pylong) @@ -334,4 +316,4 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. is only assured to produce a usable :c:type:`void` pointer for values created with :c:func:`PyLong_FromVoidPtr`. - Returns *NULL* on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index e37dec9949ab68..682160d1475c1c 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _mapping: @@ -14,8 +14,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and Return ``1`` if the object provides mapping protocol or supports slicing, and ``0`` otherwise. Note that it returns ``1`` for Python classes with a :meth:`__getitem__` method since in general case it is impossible to - determine what the type of keys it supports. This function always - succeeds. + determine what type of keys it supports. This function always succeeds. .. c:function:: Py_ssize_t PyMapping_Size(PyObject *o) @@ -29,7 +28,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key) - Return element of *o* corresponding to the string *key* or *NULL* on failure. + Return element of *o* corresponding to the string *key* or ``NULL`` on failure. This is the equivalent of the Python expression ``o[key]``. See also :c:func:`PyObject_GetItem`. @@ -38,7 +37,8 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and Map the string *key* to the value *v* in object *o*. Returns ``-1`` on failure. This is the equivalent of the Python statement ``o[key] = v``. - See also :c:func:`PyObject_SetItem`. + See also :c:func:`PyObject_SetItem`. This function *does not* steal a + reference to *v*. .. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key) @@ -79,7 +79,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_Keys(PyObject *o) On success, return a list of the keys in object *o*. On failure, return - *NULL*. + ``NULL``. .. versionchanged:: 3.7 Previously, the function returned a list or a tuple. @@ -88,7 +88,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_Values(PyObject *o) On success, return a list of the values in object *o*. On failure, return - *NULL*. + ``NULL``. .. versionchanged:: 3.7 Previously, the function returned a list or a tuple. @@ -97,7 +97,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_Items(PyObject *o) On success, return a list of the items in object *o*, where each item is a - tuple containing a key-value pair. On failure, return *NULL*. + tuple containing a key-value pair. On failure, return ``NULL``. .. versionchanged:: 3.7 Previously, the function returned a list or a tuple. diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 17ec621610b5e3..7bb0dad2b6b6d5 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _marshalling-utils: @@ -16,7 +16,7 @@ Numeric values are stored with the least significant byte first. The module supports two versions of the data format: version 0 is the historical version, version 1 shares interned strings in the file, and upon unmarshalling. Version 2 uses a binary format for floating point numbers. -*Py_MARSHAL_VERSION* indicates the current file format (currently 2). +``Py_MARSHAL_VERSION`` indicates the current file format (currently 2). .. c:function:: void PyMarshal_WriteLongToFile(long value, FILE *file, int version) @@ -43,7 +43,7 @@ The following functions allow marshalled values to be read back in. .. c:function:: long PyMarshal_ReadLongFromFile(FILE *file) - Return a C :c:type:`long` from the data stream in a :c:type:`FILE\*` opened + Return a C :c:type:`long` from the data stream in a :c:type:`FILE*` opened for reading. Only a 32-bit value can be read in using this function, regardless of the native size of :c:type:`long`. @@ -53,7 +53,7 @@ The following functions allow marshalled values to be read back in. .. c:function:: int PyMarshal_ReadShortFromFile(FILE *file) - Return a C :c:type:`short` from the data stream in a :c:type:`FILE\*` opened + Return a C :c:type:`short` from the data stream in a :c:type:`FILE*` opened for reading. Only a 16-bit value can be read in using this function, regardless of the native size of :c:type:`short`. @@ -63,16 +63,16 @@ The following functions allow marshalled values to be read back in. .. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file) - Return a Python object from the data stream in a :c:type:`FILE\*` opened for + Return a Python object from the data stream in a :c:type:`FILE*` opened for reading. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns *NULL*. + or :exc:`TypeError`) and returns ``NULL``. .. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file) - Return a Python object from the data stream in a :c:type:`FILE\*` opened for + Return a Python object from the data stream in a :c:type:`FILE*` opened for reading. Unlike :c:func:`PyMarshal_ReadObjectFromFile`, this function assumes that no further objects will be read from the file, allowing it to aggressively load file data into memory so that the de-serialization can @@ -81,7 +81,7 @@ The following functions allow marshalled values to be read back in. anything else from the file. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns *NULL*. + or :exc:`TypeError`) and returns ``NULL``. .. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) @@ -90,5 +90,5 @@ The following functions allow marshalled values to be read back in. containing *len* bytes pointed to by *data*. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns *NULL*. + or :exc:`TypeError`) and returns ``NULL``. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 65a207691b8ac8..87425bcf1e71f2 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _memory: @@ -67,7 +67,7 @@ example:: In this example, the memory request for the I/O buffer is handled by the C library allocator. The Python memory manager is involved only in the allocation -of the string object returned as a result. +of the bytes object returned as a result. In most situations, however, it is recommended to allocate memory from the Python heap specifically because the latter is under control of the Python @@ -109,10 +109,10 @@ zero bytes. .. c:function:: void* PyMem_RawMalloc(size_t n) - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the - allocated memory, or *NULL* if the request fails. + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. - Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as if ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have been initialized in any way. @@ -120,11 +120,11 @@ zero bytes. .. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. Requesting zero elements or elements of size zero bytes returns a distinct - non-*NULL* pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been + non-``NULL`` pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been called instead. .. versionadded:: 3.5 @@ -135,15 +135,15 @@ zero bytes. Resizes the memory block pointed to by *p* to *n* bytes. The contents will be unchanged to the minimum of the old and the new sizes. - If *p* is *NULL*, the call is equivalent to ``PyMem_RawMalloc(n)``; else if + If *p* is ``NULL``, the call is equivalent to ``PyMem_RawMalloc(n)``; else if *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-*NULL*. + returned pointer is non-``NULL``. - Unless *p* is *NULL*, it must have been returned by a previous call to + Unless *p* is ``NULL``, it must have been returned by a previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or :c:func:`PyMem_RawCalloc`. - If the request fails, :c:func:`PyMem_RawRealloc` returns *NULL* and *p* + If the request fails, :c:func:`PyMem_RawRealloc` returns ``NULL`` and *p* remains a valid pointer to the previous memory area. @@ -154,7 +154,7 @@ zero bytes. :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been called before, undefined behavior occurs. - If *p* is *NULL*, no operation is performed. + If *p* is ``NULL``, no operation is performed. .. _memoryinterface: @@ -180,10 +180,10 @@ The :ref:`default memory allocator ` uses the .. c:function:: void* PyMem_Malloc(size_t n) - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the - allocated memory, or *NULL* if the request fails. + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. - Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as if ``PyMem_Malloc(1)`` had been called instead. The memory will not have been initialized in any way. @@ -191,11 +191,11 @@ The :ref:`default memory allocator ` uses the .. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. Requesting zero elements or elements of size zero bytes returns a distinct - non-*NULL* pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called + non-``NULL`` pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called instead. .. versionadded:: 3.5 @@ -206,14 +206,14 @@ The :ref:`default memory allocator ` uses the Resizes the memory block pointed to by *p* to *n* bytes. The contents will be unchanged to the minimum of the old and the new sizes. - If *p* is *NULL*, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* + If *p* is ``NULL``, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-*NULL*. + returned pointer is non-``NULL``. - Unless *p* is *NULL*, it must have been returned by a previous call to + Unless *p* is ``NULL``, it must have been returned by a previous call to :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or :c:func:`PyMem_Calloc`. - If the request fails, :c:func:`PyMem_Realloc` returns *NULL* and *p* remains + If the request fails, :c:func:`PyMem_Realloc` returns ``NULL`` and *p* remains a valid pointer to the previous memory area. @@ -224,7 +224,7 @@ The :ref:`default memory allocator ` uses the :c:func:`PyMem_Calloc`. Otherwise, or if ``PyMem_Free(p)`` has been called before, undefined behavior occurs. - If *p* is *NULL*, no operation is performed. + If *p* is ``NULL``, no operation is performed. The following type-oriented macros are provided for convenience. Note that *TYPE* refers to any C type. @@ -233,15 +233,15 @@ The following type-oriented macros are provided for convenience. Note that .. c:function:: TYPE* PyMem_New(TYPE, size_t n) Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:type:`TYPE\*`. The memory will not have + memory. Returns a pointer cast to :c:type:`TYPE*`. The memory will not have been initialized in any way. .. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE\*`. On return, - *p* will be a pointer to the new memory area, or *NULL* in the event of + sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE*`. On return, + *p* will be a pointer to the new memory area, or ``NULL`` in the event of failure. This is a C preprocessor macro; *p* is always reassigned. Save the original @@ -282,10 +282,10 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Malloc(size_t n) - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the - allocated memory, or *NULL* if the request fails. + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. - Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as if ``PyObject_Malloc(1)`` had been called instead. The memory will not have been initialized in any way. @@ -293,11 +293,11 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. Requesting zero elements or elements of size zero bytes returns a distinct - non-*NULL* pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called + non-``NULL`` pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called instead. .. versionadded:: 3.5 @@ -308,14 +308,14 @@ The :ref:`default object allocator ` uses the Resizes the memory block pointed to by *p* to *n* bytes. The contents will be unchanged to the minimum of the old and the new sizes. - If *p* is *NULL*, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* + If *p* is ``NULL``, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-*NULL*. + returned pointer is non-``NULL``. - Unless *p* is *NULL*, it must have been returned by a previous call to + Unless *p* is ``NULL``, it must have been returned by a previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. - If the request fails, :c:func:`PyObject_Realloc` returns *NULL* and *p* remains + If the request fails, :c:func:`PyObject_Realloc` returns ``NULL`` and *p* remains a valid pointer to the previous memory area. @@ -326,7 +326,7 @@ The :ref:`default object allocator ` uses the :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called before, undefined behavior occurs. - If *p* is *NULL*, no operation is performed. + If *p* is ``NULL``, no operation is performed. .. _default-memory-allocators: @@ -388,7 +388,7 @@ Customize Memory Allocators Enum used to identify an allocator domain. Domains: - .. c:var:: PYMEM_DOMAIN_RAW + .. c:macro:: PYMEM_DOMAIN_RAW Functions: @@ -397,7 +397,7 @@ Customize Memory Allocators * :c:func:`PyMem_RawCalloc` * :c:func:`PyMem_RawFree` - .. c:var:: PYMEM_DOMAIN_MEM + .. c:macro:: PYMEM_DOMAIN_MEM Functions: @@ -406,7 +406,7 @@ Customize Memory Allocators * :c:func:`PyMem_Calloc` * :c:func:`PyMem_Free` - .. c:var:: PYMEM_DOMAIN_OBJ + .. c:macro:: PYMEM_DOMAIN_OBJ Functions: @@ -424,7 +424,7 @@ Customize Memory Allocators Set the memory block allocator of the specified domain. - The new allocator must return a distinct non-NULL pointer when requesting + The new allocator must return a distinct non-``NULL`` pointer when requesting zero bytes. For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be @@ -472,7 +472,7 @@ Customize Memory Allocators if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and :c:data:`PYMEM_DOMAIN_MEM` domains are called. - .. versionchanged:: 3.8.0 + .. versionchanged:: 3.8 Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` @@ -519,11 +519,11 @@ Customize pymalloc Arena Allocator | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | +--------------------------------------------------+---------------------------------------+ -.. c:function:: PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) +.. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) Get the arena allocator. -.. c:function:: PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) +.. c:function:: void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) Set the arena allocator. @@ -533,7 +533,7 @@ tracemalloc C API .. versionadded:: 3.7 -.. c:function: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) +.. c:function:: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) Track an allocated memory block in the :mod:`tracemalloc` module. @@ -542,7 +542,7 @@ tracemalloc C API If memory block is already tracked, update the existing trace. -.. c:function: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) +.. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) Untrack an allocated memory block in the :mod:`tracemalloc` module. Do nothing if the block was not tracked. diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index 9f6bfd751ade5e..de429e5c11dc76 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _memoryview-objects: @@ -57,7 +57,7 @@ any other object. .. c:function:: Py_buffer *PyMemoryView_GET_BASE(PyObject *mview) Return either a pointer to the exporting object that the memoryview is based - on or *NULL* if the memoryview has been created by one of the functions + on or ``NULL`` if the memoryview has been created by one of the functions :c:func:`PyMemoryView_FromMemory` or :c:func:`PyMemoryView_FromBuffer`. *mview* **must** be a memoryview instance. diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst index 7a2a84fe110dcf..0a5341cbbdf152 100644 --- a/Doc/c-api/method.rst +++ b/Doc/c-api/method.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _instancemethod-objects: @@ -21,7 +21,7 @@ to bind a :c:data:`PyCFunction` to a class object. It replaces the former call .. c:function:: int PyInstanceMethod_Check(PyObject *o) Return true if *o* is an instance method object (has type - :c:data:`PyInstanceMethod_Type`). The parameter must not be *NULL*. + :c:data:`PyInstanceMethod_Type`). The parameter must not be ``NULL``. .. c:function:: PyObject* PyInstanceMethod_New(PyObject *func) @@ -64,14 +64,14 @@ no longer available. .. c:function:: int PyMethod_Check(PyObject *o) Return true if *o* is a method object (has type :c:data:`PyMethod_Type`). The - parameter must not be *NULL*. + parameter must not be ``NULL``. .. c:function:: PyObject* PyMethod_New(PyObject *func, PyObject *self) Return a new method object, with *func* being any callable object and *self* the instance the method should be bound. *func* is the function that will - be called when the method is called. *self* must not be *NULL*. + be called when the method is called. *self* must not be ``NULL``. .. c:function:: PyObject* PyMethod_Function(PyObject *meth) @@ -92,9 +92,3 @@ no longer available. .. c:function:: PyObject* PyMethod_GET_SELF(PyObject *meth) Macro version of :c:func:`PyMethod_Self` which avoids error checking. - - -.. c:function:: int PyMethod_ClearFreeList() - - Clear the free list. Return the total number of freed items. - diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 017b656854a8cd..6e9474bfa40eba 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _moduleobjects: @@ -61,7 +61,7 @@ Module Objects Return the dictionary object that implements *module*'s namespace; this object is the same as the :attr:`~object.__dict__` attribute of the module object. If *module* is not a module object (or a subtype of a module object), - :exc:`SystemError` is raised and *NULL* is returned. + :exc:`SystemError` is raised and ``NULL`` is returned. It is recommended extensions use other :c:func:`PyModule_\*` and :c:func:`PyObject_\*` functions rather than directly manipulate a module's @@ -75,7 +75,7 @@ Module Objects single: SystemError (built-in exception) Return *module*'s :attr:`__name__` value. If the module does not provide one, - or if it is not a string, :exc:`SystemError` is raised and *NULL* is returned. + or if it is not a string, :exc:`SystemError` is raised and ``NULL`` is returned. .. versionadded:: 3.3 @@ -88,14 +88,14 @@ Module Objects .. c:function:: void* PyModule_GetState(PyObject *module) Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or *NULL*. See + allocated at module creation time, or ``NULL``. See :c:member:`PyModuleDef.m_size`. .. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) Return a pointer to the :c:type:`PyModuleDef` struct from which the module was - created, or *NULL* if the module wasn't created from a definition. + created, or ``NULL`` if the module wasn't created from a definition. .. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) @@ -106,7 +106,7 @@ Module Objects Return the name of the file from which *module* was loaded using *module*'s :attr:`__file__` attribute. If this is not defined, or if it is not a - unicode string, raise :exc:`SystemError` and return *NULL*; otherwise return + unicode string, raise :exc:`SystemError` and return ``NULL``; otherwise return a reference to a Unicode object. .. versionadded:: 3.2 @@ -153,7 +153,7 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: const char *m_doc Docstring for the module; usually a docstring variable created with - :c:func:`PyDoc_STRVAR` is used. + :c:macro:`PyDoc_STRVAR` is used. .. c:member:: Py_ssize_t m_size @@ -178,17 +178,17 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: PyMethodDef* m_methods A pointer to a table of module-level functions, described by - :c:type:`PyMethodDef` values. Can be *NULL* if no functions are present. + :c:type:`PyMethodDef` values. Can be ``NULL`` if no functions are present. .. c:member:: PyModuleDef_Slot* m_slots An array of slot definitions for multi-phase initialization, terminated by a ``{0, NULL}`` entry. - When using single-phase initialization, *m_slots* must be *NULL*. + When using single-phase initialization, *m_slots* must be ``NULL``. .. versionchanged:: 3.5 - Prior to version 3.5, this member was always set to *NULL*, + Prior to version 3.5, this member was always set to ``NULL``, and was defined as: .. c:member:: inquiry m_reload @@ -196,23 +196,47 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: traverseproc m_traverse A traversal function to call during GC traversal of the module object, or - *NULL* if not needed. This function may be called before module state - is allocated (:c:func:`PyModule_GetState()` may return `NULL`), - and before the :c:member:`Py_mod_exec` function is executed. + ``NULL`` if not needed. + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if :c:member:`m_size` is greater + than 0 and the module state (as returned by :c:func:`PyModule_GetState`) + is ``NULL``. + + .. versionchanged:: 3.9 + No longer called before the module state is allocated. .. c:member:: inquiry m_clear A clear function to call during GC clearing of the module object, or - *NULL* if not needed. This function may be called before module state - is allocated (:c:func:`PyModule_GetState()` may return `NULL`), - and before the :c:member:`Py_mod_exec` function is executed. + ``NULL`` if not needed. + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if :c:member:`m_size` is greater + than 0 and the module state (as returned by :c:func:`PyModule_GetState`) + is ``NULL``. + + .. versionchanged:: 3.9 + No longer called before the module state is allocated. .. c:member:: freefunc m_free - A function to call during deallocation of the module object, or *NULL* if - not needed. This function may be called before module state - is allocated (:c:func:`PyModule_GetState()` may return `NULL`), - and before the :c:member:`Py_mod_exec` function is executed. + A function to call during deallocation of the module object, or ``NULL`` + if not needed. + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if :c:member:`m_size` is greater + than 0 and the module state (as returned by :c:func:`PyModule_GetState`) + is ``NULL``. + + .. versionchanged:: 3.9 + No longer called before the module state is allocated. Single-phase initialization ........................... @@ -278,7 +302,7 @@ instance must be initialized with the following function: Ensures a module definition is a properly initialized Python object that correctly reports its type and reference count. - Returns *def* cast to ``PyObject*``, or *NULL* if an error occurred. + Returns *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. .. versionadded:: 3.5 @@ -301,7 +325,7 @@ The *m_slots* array must be terminated by a slot with id 0. The available slot types are: -.. c:var:: Py_mod_create +.. c:macro:: Py_mod_create Specifies a function that is called to create the module object itself. The *value* pointer of this slot must point to a function of the signature: @@ -311,7 +335,7 @@ The available slot types are: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. It should return a new module object, or set an error - and return *NULL*. + and return ``NULL``. This function should be kept minimal. In particular, it should not call arbitrary Python code, as trying to import the same module again may @@ -330,10 +354,10 @@ The available slot types are: :c:type:`PyModule_Type`. Any type can be used, as long as it supports setting and getting import-related attributes. However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-*NULL* ``m_traverse``, ``m_clear``, + ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. -.. c:var:: Py_mod_exec +.. c:macro:: Py_mod_exec Specifies a function that is called to *execute* the module. This is equivalent to executing the code of a Python module: typically, @@ -394,7 +418,7 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and .. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) - Add the functions from the *NULL* terminated *functions* array to *module*. + Add the functions from the ``NULL`` terminated *functions* array to *module*. Refer to the :c:type:`PyMethodDef` documentation for details on individual entries (due to the lack of a shared module namespace, module level "functions" implemented in C typically receive the module as their first @@ -417,7 +441,22 @@ state: Add an object to *module* as *name*. This is a convenience function which can be used from the module's initialization function. This steals a reference to - *value*. Return ``-1`` on error, ``0`` on success. + *value* on success. Return ``-1`` on error, ``0`` on success. + + .. note:: + + Unlike other functions that steal references, ``PyModule_AddObject()`` only + decrements the reference count of *value* **on success**. + + This means that its return value must be checked, and calling code must + :c:func:`Py_DECREF` *value* manually on error. Example usage:: + + Py_INCREF(spam); + if (PyModule_AddObject(module, "spam", spam) < 0) { + Py_DECREF(module); + Py_DECREF(spam); + return NULL; + } .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) @@ -430,7 +469,7 @@ state: Add a string constant to *module* as *name*. This convenience function can be used from the module's initialization function. The string *value* must be - *NULL*-terminated. Return ``-1`` on error, ``0`` on success. + ``NULL``-terminated. Return ``-1`` on error, ``0`` on success. .. c:function:: int PyModule_AddIntMacro(PyObject *module, macro) @@ -445,6 +484,16 @@ state: Add a string constant to *module*. +.. c:function:: int PyModule_AddType(PyObject *module, PyTypeObject *type) + + Add a type object to *module*. + The type object is finalized by calling internally :c:func:`PyType_Ready`. + The name of the type object is taken from the last component of + :c:member:`~PyTypeObject.tp_name` after dot. + Return ``-1`` on error, ``0`` on success. + + .. versionadded:: 3.9 + Module lookup ^^^^^^^^^^^^^ @@ -461,7 +510,7 @@ since multiple such modules can be created from a single definition. Returns the module object that was created from *def* for the current interpreter. This method requires that the module object has been attached to the interpreter state with :c:func:`PyState_AddModule` beforehand. In case the corresponding module object is not - found or has not been attached to the interpreter state yet, it returns *NULL*. + found or has not been attached to the interpreter state yet, it returns ``NULL``. .. c:function:: int PyState_AddModule(PyObject *module, PyModuleDef *def) @@ -470,10 +519,25 @@ since multiple such modules can be created from a single definition. Only effective on modules created using single-phase initialization. + Python calls ``PyState_AddModule`` automatically after importing a module, + so it is unnecessary (but harmless) to call it from module initialization + code. An explicit call is needed only if the module's own init code + subsequently calls ``PyState_FindModule``. + The function is mainly intended for implementing alternative import + mechanisms (either by calling it directly, or by referring to its + implementation for details of the required state updates). + + The caller must hold the GIL. + + Return 0 on success or -1 on failure. + .. versionadded:: 3.3 .. c:function:: int PyState_RemoveModule(PyModuleDef *def) Removes the module object created from *def* from the interpreter state. + Return 0 on success or -1 on failure. + + The caller must hold the GIL. .. versionadded:: 3.3 diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst index 45568fe657b938..26d2b7aab201ba 100644 --- a/Doc/c-api/none.rst +++ b/Doc/c-api/none.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _noneobject: diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 9fb220b192b9a6..37979bb506bcfa 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _number: @@ -17,25 +17,25 @@ Number Protocol .. c:function:: PyObject* PyNumber_Add(PyObject *o1, PyObject *o2) - Returns the result of adding *o1* and *o2*, or *NULL* on failure. This is the + Returns the result of adding *o1* and *o2*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 + o2``. .. c:function:: PyObject* PyNumber_Subtract(PyObject *o1, PyObject *o2) - Returns the result of subtracting *o2* from *o1*, or *NULL* on failure. This is + Returns the result of subtracting *o2* from *o1*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 - o2``. .. c:function:: PyObject* PyNumber_Multiply(PyObject *o1, PyObject *o2) - Returns the result of multiplying *o1* and *o2*, or *NULL* on failure. This is + Returns the result of multiplying *o1* and *o2*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 * o2``. .. c:function:: PyObject* PyNumber_MatrixMultiply(PyObject *o1, PyObject *o2) - Returns the result of matrix multiplication on *o1* and *o2*, or *NULL* on + Returns the result of matrix multiplication on *o1* and *o2*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 @ o2``. .. versionadded:: 3.5 @@ -43,14 +43,14 @@ Number Protocol .. c:function:: PyObject* PyNumber_FloorDivide(PyObject *o1, PyObject *o2) - Return the floor of *o1* divided by *o2*, or *NULL* on failure. This is + Return the floor of *o1* divided by *o2*, or ``NULL`` on failure. This is equivalent to the "classic" division of integers. .. c:function:: PyObject* PyNumber_TrueDivide(PyObject *o1, PyObject *o2) Return a reasonable approximation for the mathematical value of *o1* divided by - *o2*, or *NULL* on failure. The return value is "approximate" because binary + *o2*, or ``NULL`` on failure. The return value is "approximate" because binary floating point numbers are approximate; it is not possible to represent all real numbers in base two. This function can return a floating point value when passed two integers. @@ -58,7 +58,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Remainder(PyObject *o1, PyObject *o2) - Returns the remainder of dividing *o1* by *o2*, or *NULL* on failure. This is + Returns the remainder of dividing *o1* by *o2*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 % o2``. @@ -66,7 +66,7 @@ Number Protocol .. index:: builtin: divmod - See the built-in function :func:`divmod`. Returns *NULL* on failure. This is + See the built-in function :func:`divmod`. Returns ``NULL`` on failure. This is the equivalent of the Python expression ``divmod(o1, o2)``. @@ -74,21 +74,21 @@ Number Protocol .. index:: builtin: pow - See the built-in function :func:`pow`. Returns *NULL* on failure. This is the + See the built-in function :func:`pow`. Returns ``NULL`` on failure. This is the equivalent of the Python expression ``pow(o1, o2, o3)``, where *o3* is optional. - If *o3* is to be ignored, pass :c:data:`Py_None` in its place (passing *NULL* for + If *o3* is to be ignored, pass :c:data:`Py_None` in its place (passing ``NULL`` for *o3* would cause an illegal memory access). .. c:function:: PyObject* PyNumber_Negative(PyObject *o) - Returns the negation of *o* on success, or *NULL* on failure. This is the + Returns the negation of *o* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``-o``. .. c:function:: PyObject* PyNumber_Positive(PyObject *o) - Returns *o* on success, or *NULL* on failure. This is the equivalent of the + Returns *o* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``+o``. @@ -96,70 +96,70 @@ Number Protocol .. index:: builtin: abs - Returns the absolute value of *o*, or *NULL* on failure. This is the equivalent + Returns the absolute value of *o*, or ``NULL`` on failure. This is the equivalent of the Python expression ``abs(o)``. .. c:function:: PyObject* PyNumber_Invert(PyObject *o) - Returns the bitwise negation of *o* on success, or *NULL* on failure. This is + Returns the bitwise negation of *o* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``~o``. .. c:function:: PyObject* PyNumber_Lshift(PyObject *o1, PyObject *o2) - Returns the result of left shifting *o1* by *o2* on success, or *NULL* on + Returns the result of left shifting *o1* by *o2* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 << o2``. .. c:function:: PyObject* PyNumber_Rshift(PyObject *o1, PyObject *o2) - Returns the result of right shifting *o1* by *o2* on success, or *NULL* on + Returns the result of right shifting *o1* by *o2* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 >> o2``. .. c:function:: PyObject* PyNumber_And(PyObject *o1, PyObject *o2) - Returns the "bitwise and" of *o1* and *o2* on success and *NULL* on failure. + Returns the "bitwise and" of *o1* and *o2* on success and ``NULL`` on failure. This is the equivalent of the Python expression ``o1 & o2``. .. c:function:: PyObject* PyNumber_Xor(PyObject *o1, PyObject *o2) - Returns the "bitwise exclusive or" of *o1* by *o2* on success, or *NULL* on + Returns the "bitwise exclusive or" of *o1* by *o2* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 ^ o2``. .. c:function:: PyObject* PyNumber_Or(PyObject *o1, PyObject *o2) - Returns the "bitwise or" of *o1* and *o2* on success, or *NULL* on failure. + Returns the "bitwise or" of *o1* and *o2* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 | o2``. .. c:function:: PyObject* PyNumber_InPlaceAdd(PyObject *o1, PyObject *o2) - Returns the result of adding *o1* and *o2*, or *NULL* on failure. The operation + Returns the result of adding *o1* and *o2*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 += o2``. .. c:function:: PyObject* PyNumber_InPlaceSubtract(PyObject *o1, PyObject *o2) - Returns the result of subtracting *o2* from *o1*, or *NULL* on failure. The + Returns the result of subtracting *o2* from *o1*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 -= o2``. .. c:function:: PyObject* PyNumber_InPlaceMultiply(PyObject *o1, PyObject *o2) - Returns the result of multiplying *o1* and *o2*, or *NULL* on failure. The + Returns the result of multiplying *o1* and *o2*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 *= o2``. .. c:function:: PyObject* PyNumber_InPlaceMatrixMultiply(PyObject *o1, PyObject *o2) - Returns the result of matrix multiplication on *o1* and *o2*, or *NULL* on + Returns the result of matrix multiplication on *o1* and *o2*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 @= o2``. @@ -168,7 +168,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_InPlaceFloorDivide(PyObject *o1, PyObject *o2) - Returns the mathematical floor of dividing *o1* by *o2*, or *NULL* on failure. + Returns the mathematical floor of dividing *o1* by *o2*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 //= o2``. @@ -176,7 +176,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_InPlaceTrueDivide(PyObject *o1, PyObject *o2) Return a reasonable approximation for the mathematical value of *o1* divided by - *o2*, or *NULL* on failure. The return value is "approximate" because binary + *o2*, or ``NULL`` on failure. The return value is "approximate" because binary floating point numbers are approximate; it is not possible to represent all real numbers in base two. This function can return a floating point value when passed two integers. The operation is done *in-place* when *o1* supports it. @@ -184,7 +184,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_InPlaceRemainder(PyObject *o1, PyObject *o2) - Returns the remainder of dividing *o1* by *o2*, or *NULL* on failure. The + Returns the remainder of dividing *o1* by *o2*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 %= o2``. @@ -193,44 +193,44 @@ Number Protocol .. index:: builtin: pow - See the built-in function :func:`pow`. Returns *NULL* on failure. The operation + See the built-in function :func:`pow`. Returns ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 **= o2`` when o3 is :c:data:`Py_None`, or an in-place variant of ``pow(o1, o2, o3)`` otherwise. If *o3* is to be ignored, pass :c:data:`Py_None` - in its place (passing *NULL* for *o3* would cause an illegal memory access). + in its place (passing ``NULL`` for *o3* would cause an illegal memory access). .. c:function:: PyObject* PyNumber_InPlaceLshift(PyObject *o1, PyObject *o2) - Returns the result of left shifting *o1* by *o2* on success, or *NULL* on + Returns the result of left shifting *o1* by *o2* on success, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 <<= o2``. .. c:function:: PyObject* PyNumber_InPlaceRshift(PyObject *o1, PyObject *o2) - Returns the result of right shifting *o1* by *o2* on success, or *NULL* on + Returns the result of right shifting *o1* by *o2* on success, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 >>= o2``. .. c:function:: PyObject* PyNumber_InPlaceAnd(PyObject *o1, PyObject *o2) - Returns the "bitwise and" of *o1* and *o2* on success and *NULL* on failure. The + Returns the "bitwise and" of *o1* and *o2* on success and ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 &= o2``. .. c:function:: PyObject* PyNumber_InPlaceXor(PyObject *o1, PyObject *o2) - Returns the "bitwise exclusive or" of *o1* by *o2* on success, or *NULL* on + Returns the "bitwise exclusive or" of *o1* by *o2* on success, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 ^= o2``. .. c:function:: PyObject* PyNumber_InPlaceOr(PyObject *o1, PyObject *o2) - Returns the "bitwise or" of *o1* and *o2* on success, or *NULL* on failure. The + Returns the "bitwise or" of *o1* and *o2* on success, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 |= o2``. @@ -239,7 +239,7 @@ Number Protocol .. index:: builtin: int - Returns the *o* converted to an integer object on success, or *NULL* on + Returns the *o* converted to an integer object on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``int(o)``. @@ -247,15 +247,19 @@ Number Protocol .. index:: builtin: float - Returns the *o* converted to a float object on success, or *NULL* on failure. + Returns the *o* converted to a float object on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``float(o)``. .. c:function:: PyObject* PyNumber_Index(PyObject *o) - Returns the *o* converted to a Python int on success or *NULL* with a + Returns the *o* converted to a Python int on success or ``NULL`` with a :exc:`TypeError` exception raised on failure. + .. versionchanged:: 3.10 + The result always has exact type :class:`int`. Previously, the result + could have been an instance of a subclass of ``int``. + .. c:function:: PyObject* PyNumber_ToBase(PyObject *n, int base) @@ -274,9 +278,9 @@ Number Protocol If *o* can be converted to a Python int but the attempt to convert to a Py_ssize_t value would raise an :exc:`OverflowError`, then the *exc* argument is the type of exception that will be raised (usually - :exc:`IndexError` or :exc:`OverflowError`). If *exc* is *NULL*, then the - exception is cleared and the value is clipped to *PY_SSIZE_T_MIN* for a negative - integer or *PY_SSIZE_T_MAX* for a positive integer. + :exc:`IndexError` or :exc:`OverflowError`). If *exc* is ``NULL``, then the + exception is cleared and the value is clipped to ``PY_SSIZE_T_MIN`` for a negative + integer or ``PY_SSIZE_T_MAX`` for a positive integer. .. c:function:: int PyIndex_Check(PyObject *o) diff --git a/Doc/c-api/objbuffer.rst b/Doc/c-api/objbuffer.rst deleted file mode 100644 index 3572564b13e9b3..00000000000000 --- a/Doc/c-api/objbuffer.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. highlightlang:: c - -Old Buffer Protocol -------------------- - -.. deprecated:: 3.0 - -These functions were part of the "old buffer protocol" API in Python 2. -In Python 3, this protocol doesn't exist anymore but the functions are still -exposed to ease porting 2.x code. They act as a compatibility wrapper -around the :ref:`new buffer protocol `, but they don't give -you control over the lifetime of the resources acquired when a buffer is -exported. - -Therefore, it is recommended that you call :c:func:`PyObject_GetBuffer` -(or the ``y*`` or ``w*`` :ref:`format codes ` with the -:c:func:`PyArg_ParseTuple` family of functions) to get a buffer view over -an object, and :c:func:`PyBuffer_Release` when the buffer view can be released. - - -.. c:function:: int PyObject_AsCharBuffer(PyObject *obj, const char **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a read-only memory location usable as character-based - input. The *obj* argument must support the single-segment character buffer - interface. On success, returns ``0``, sets *buffer* to the memory location - and *buffer_len* to the buffer length. Returns ``-1`` and sets a - :exc:`TypeError` on error. - - -.. c:function:: int PyObject_AsReadBuffer(PyObject *obj, const void **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a read-only memory location containing arbitrary data. - The *obj* argument must support the single-segment readable buffer - interface. On success, returns ``0``, sets *buffer* to the memory location - and *buffer_len* to the buffer length. Returns ``-1`` and sets a - :exc:`TypeError` on error. - - -.. c:function:: int PyObject_CheckReadBuffer(PyObject *o) - - Returns ``1`` if *o* supports the single-segment readable buffer interface. - Otherwise returns ``0``. This function always succeeds. - - Note that this function tries to get and release a buffer, and exceptions - which occur while calling corresponding functions will get suppressed. - To get error reporting use :c:func:`PyObject_GetBuffer()` instead. - - -.. c:function:: int PyObject_AsWriteBuffer(PyObject *obj, void **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a writable memory location. The *obj* argument must - support the single-segment, character buffer interface. On success, - returns ``0``, sets *buffer* to the memory location and *buffer_len* to the - buffer length. Returns ``-1`` and sets a :exc:`TypeError` on error. - diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a64ff2e6b58b4c..a387b4a2df1342 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _object: @@ -53,14 +53,14 @@ Object Protocol .. c:function:: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name) Retrieve an attribute named *attr_name* from object *o*. Returns the attribute - value on success, or *NULL* on failure. This is the equivalent of the Python + value on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o.attr_name``. .. c:function:: PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name) Retrieve an attribute named *attr_name* from object *o*. Returns the attribute - value on success, or *NULL* on failure. This is the equivalent of the Python + value on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o.attr_name``. @@ -81,7 +81,7 @@ Object Protocol return ``0`` on success. This is the equivalent of the Python statement ``o.attr_name = v``. - If *v* is *NULL*, the attribute is deleted, however this feature is + If *v* is ``NULL``, the attribute is deleted, however this feature is deprecated in favour of using :c:func:`PyObject_DelAttr`. @@ -92,7 +92,7 @@ Object Protocol return ``0`` on success. This is the equivalent of the Python statement ``o.attr_name = v``. - If *v* is *NULL*, the attribute is deleted, however this feature is + If *v* is ``NULL``, the attribute is deleted, however this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. @@ -128,7 +128,7 @@ Object Protocol .. versionadded:: 3.3 -.. c:function:: int PyObject_GenericSetDict(PyObject *o, void *context) +.. c:function:: int PyObject_GenericSetDict(PyObject *o, PyObject *value, void *context) A generic implementation for the setter of a ``__dict__`` descriptor. This implementation does not allow the dictionary to be deleted. @@ -143,7 +143,7 @@ Object Protocol :const:`Py_NE`, :const:`Py_GT`, or :const:`Py_GE`, corresponding to ``<``, ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. This is the equivalent of the Python expression ``o1 op o2``, where ``op`` is the operator corresponding - to *opid*. Returns the value of the comparison on success, or *NULL* on failure. + to *opid*. Returns the value of the comparison on success, or ``NULL`` on failure. .. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid) @@ -165,7 +165,7 @@ Object Protocol .. index:: builtin: repr Compute a string representation of object *o*. Returns the string - representation on success, *NULL* on failure. This is the equivalent of the + representation on success, ``NULL`` on failure. This is the equivalent of the Python expression ``repr(o)``. Called by the :func:`repr` built-in function. .. versionchanged:: 3.4 @@ -188,7 +188,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Str(PyObject *o) Compute a string representation of object *o*. Returns the string - representation on success, *NULL* on failure. This is the equivalent of the + representation on success, ``NULL`` on failure. This is the equivalent of the Python expression ``str(o)``. Called by the :func:`str` built-in function and, therefore, by the :func:`print` function. @@ -196,11 +196,12 @@ Object Protocol This function now includes a debug assertion to help ensure that it does not silently discard an active exception. + .. c:function:: PyObject* PyObject_Bytes(PyObject *o) .. index:: builtin: bytes - Compute a bytes representation of object *o*. *NULL* is returned on + Compute a bytes representation of object *o*. ``NULL`` is returned on failure and a bytes object on success. This is equivalent to the Python expression ``bytes(o)``, when *o* is not an integer. Unlike ``bytes(o)``, a TypeError is raised when *o* is an integer instead of a zero-initialized @@ -247,94 +248,6 @@ Object Protocol of base classes). -.. c:function:: int PyCallable_Check(PyObject *o) - - Determine if the object *o* is callable. Return ``1`` if the object is callable - and ``0`` otherwise. This function always succeeds. - - -.. c:function:: PyObject* PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs) - - Call a callable Python object *callable*, with arguments given by the - tuple *args*, and named arguments given by the dictionary *kwargs*. - - *args* must not be *NULL*, use an empty tuple if no arguments are needed. - If no named arguments are needed, *kwargs* can be *NULL*. - - Returns the result of the call on success, or *NULL* on failure. - - This is the equivalent of the Python expression: - ``callable(*args, **kwargs)``. - - -.. c:function:: PyObject* PyObject_CallObject(PyObject *callable, PyObject *args) - - Call a callable Python object *callable*, with arguments given by the - tuple *args*. If no arguments are needed, then *args* can be *NULL*. - - Returns the result of the call on success, or *NULL* on failure. - - This is the equivalent of the Python expression: ``callable(*args)``. - - -.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, const char *format, ...) - - Call a callable Python object *callable*, with a variable number of C arguments. - The C arguments are described using a :c:func:`Py_BuildValue` style format - string. The format can be *NULL*, indicating that no arguments are provided. - - Returns the result of the call on success, or *NULL* on failure. - - This is the equivalent of the Python expression: ``callable(*args)``. - - Note that if you only pass :c:type:`PyObject \*` args, - :c:func:`PyObject_CallFunctionObjArgs` is a faster alternative. - - .. versionchanged:: 3.4 - The type of *format* was changed from ``char *``. - - -.. c:function:: PyObject* PyObject_CallMethod(PyObject *obj, const char *name, const char *format, ...) - - Call the method named *name* of object *obj* with a variable number of C - arguments. The C arguments are described by a :c:func:`Py_BuildValue` format - string that should produce a tuple. - - The format can be *NULL*, indicating that no arguments are provided. - - Returns the result of the call on success, or *NULL* on failure. - - This is the equivalent of the Python expression: - ``obj.name(arg1, arg2, ...)``. - - Note that if you only pass :c:type:`PyObject \*` args, - :c:func:`PyObject_CallMethodObjArgs` is a faster alternative. - - .. versionchanged:: 3.4 - The types of *name* and *format* were changed from ``char *``. - - -.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ..., NULL) - - Call a callable Python object *callable*, with a variable number of - :c:type:`PyObject\*` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. - - Returns the result of the call on success, or *NULL* on failure. - - This is the equivalent of the Python expression: - ``callable(arg1, arg2, ...)``. - - -.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ..., NULL) - - Calls a method of the Python object *obj*, where the name of the method is given as a - Python string object in *name*. It is called with a variable number of - :c:type:`PyObject\*` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. Returns the result of the call on success, or - *NULL* on failure. - - .. c:function:: Py_hash_t PyObject_Hash(PyObject *o) .. index:: builtin: hash @@ -373,19 +286,19 @@ Object Protocol .. index:: builtin: type - When *o* is non-*NULL*, returns a type object corresponding to the object type - of object *o*. On failure, raises :exc:`SystemError` and returns *NULL*. This + When *o* is non-``NULL``, returns a type object corresponding to the object type + of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This is equivalent to the Python expression ``type(o)``. This function increments the reference count of the return value. There's really no reason to use this function instead of the common expression ``o->ob_type``, which returns a - pointer of type :c:type:`PyTypeObject\*`, except when the incremented reference + pointer of type :c:type:`PyTypeObject*`, except when the incremented reference count is needed. .. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) Return true if the object *o* is of type *type* or a subtype of *type*. Both - parameters must be non-*NULL*. + parameters must be non-``NULL``. .. c:function:: Py_ssize_t PyObject_Size(PyObject *o) @@ -410,7 +323,7 @@ Object Protocol .. c:function:: PyObject* PyObject_GetItem(PyObject *o, PyObject *key) - Return element of *o* corresponding to the object *key* or *NULL* on failure. + Return element of *o* corresponding to the object *key* or ``NULL`` on failure. This is the equivalent of the Python expression ``o[key]``. @@ -418,7 +331,8 @@ Object Protocol Map the object *key* to the value *v*. Raise an exception and return ``-1`` on failure; return ``0`` on success. This is the - equivalent of the Python statement ``o[key] = v``. + equivalent of the Python statement ``o[key] = v``. This function *does + not* steal a reference to *v*. .. c:function:: int PyObject_DelItem(PyObject *o, PyObject *key) @@ -430,15 +344,15 @@ Object Protocol .. c:function:: PyObject* PyObject_Dir(PyObject *o) This is equivalent to the Python expression ``dir(o)``, returning a (possibly - empty) list of strings appropriate for the object argument, or *NULL* if there - was an error. If the argument is *NULL*, this is like the Python ``dir()``, + empty) list of strings appropriate for the object argument, or ``NULL`` if there + was an error. If the argument is ``NULL``, this is like the Python ``dir()``, returning the names of the current locals; in this case, if no execution frame - is active then *NULL* is returned but :c:func:`PyErr_Occurred` will return false. + is active then ``NULL`` is returned but :c:func:`PyErr_Occurred` will return false. .. c:function:: PyObject* PyObject_GetIter(PyObject *o) This is equivalent to the Python expression ``iter(o)``. It returns a new iterator for the object argument, or the object itself if the object is already - an iterator. Raises :exc:`TypeError` and returns *NULL* if the object cannot be + an iterator. Raises :exc:`TypeError` and returns ``NULL`` if the object cannot be iterated. diff --git a/Doc/c-api/objimpl.rst b/Doc/c-api/objimpl.rst index 7023e519eda7b3..8bd8c107c98bdf 100644 --- a/Doc/c-api/objimpl.rst +++ b/Doc/c-api/objimpl.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _newtypes: diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 225a1feb250684..0df12c67f40bc3 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _countingrefs: @@ -13,22 +13,22 @@ objects. .. c:function:: void Py_INCREF(PyObject *o) - Increment the reference count for object *o*. The object must not be *NULL*; if - you aren't sure that it isn't *NULL*, use :c:func:`Py_XINCREF`. + Increment the reference count for object *o*. The object must not be ``NULL``; if + you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. .. c:function:: void Py_XINCREF(PyObject *o) - Increment the reference count for object *o*. The object may be *NULL*, in + Increment the reference count for object *o*. The object may be ``NULL``, in which case the macro has no effect. .. c:function:: void Py_DECREF(PyObject *o) - Decrement the reference count for object *o*. The object must not be *NULL*; if - you aren't sure that it isn't *NULL*, use :c:func:`Py_XDECREF`. If the reference + Decrement the reference count for object *o*. The object must not be ``NULL``; if + you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`. If the reference count reaches zero, the object's type's deallocation function (which must not be - *NULL*) is invoked. + ``NULL``) is invoked. .. warning:: @@ -44,18 +44,18 @@ objects. .. c:function:: void Py_XDECREF(PyObject *o) - Decrement the reference count for object *o*. The object may be *NULL*, in + Decrement the reference count for object *o*. The object may be ``NULL``, in which case the macro has no effect; otherwise the effect is the same as for :c:func:`Py_DECREF`, and the same warning applies. .. c:function:: void Py_CLEAR(PyObject *o) - Decrement the reference count for object *o*. The object may be *NULL*, in + Decrement the reference count for object *o*. The object may be ``NULL``, in which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, except that the argument is also set to *NULL*. The warning + :c:func:`Py_DECREF`, except that the argument is also set to ``NULL``. The warning for :c:func:`Py_DECREF` does not apply with respect to the object passed because - the macro carefully uses a temporary variable and sets the argument to *NULL* + the macro carefully uses a temporary variable and sets the argument to ``NULL`` before decrementing its reference count. It is a good idea to use this macro whenever decrementing the reference diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 96893652f730cc..9207d86012c8b3 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -1,38 +1,64 @@ -.. highlightlang:: c +.. highlight:: c .. _reflection: Reflection ========== -.. c:function:: PyObject* PyEval_GetBuiltins() +.. c:function:: PyObject* PyEval_GetBuiltins(void) Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is currently executing. -.. c:function:: PyObject* PyEval_GetLocals() +.. c:function:: PyObject* PyEval_GetLocals(void) Return a dictionary of the local variables in the current execution frame, - or *NULL* if no frame is currently executing. + or ``NULL`` if no frame is currently executing. -.. c:function:: PyObject* PyEval_GetGlobals() +.. c:function:: PyObject* PyEval_GetGlobals(void) Return a dictionary of the global variables in the current execution frame, - or *NULL* if no frame is currently executing. + or ``NULL`` if no frame is currently executing. -.. c:function:: PyFrameObject* PyEval_GetFrame() +.. c:function:: PyFrameObject* PyEval_GetFrame(void) - Return the current thread state's frame, which is *NULL* if no frame is + Return the current thread state's frame, which is ``NULL`` if no frame is currently executing. + See also :c:func:`PyThreadState_GetFrame`. + + +.. c:function:: int PyFrame_GetBack(PyFrameObject *frame) + + Get the *frame* next outer frame. + + Return a strong reference, or ``NULL`` if *frame* has no outer frame. + + *frame* must not be ``NULL``. + + .. versionadded:: 3.9 + + +.. c:function:: int PyFrame_GetCode(PyFrameObject *frame) + + Get the *frame* code. + + Return a strong reference. + + *frame* must not be ``NULL``. The result (frame code) cannot be ``NULL``. + + .. versionadded:: 3.9 + .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) Return the line number that *frame* is currently executing. + *frame* must not be ``NULL``. + .. c:function:: const char* PyEval_GetFuncName(PyObject *func) diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index 6d22f35e22b1f2..65818859041179 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _sequence: @@ -26,39 +26,39 @@ Sequence Protocol .. c:function:: PyObject* PySequence_Concat(PyObject *o1, PyObject *o2) - Return the concatenation of *o1* and *o2* on success, and *NULL* on failure. + Return the concatenation of *o1* and *o2* on success, and ``NULL`` on failure. This is the equivalent of the Python expression ``o1 + o2``. .. c:function:: PyObject* PySequence_Repeat(PyObject *o, Py_ssize_t count) - Return the result of repeating sequence object *o* *count* times, or *NULL* on + Return the result of repeating sequence object *o* *count* times, or ``NULL`` on failure. This is the equivalent of the Python expression ``o * count``. .. c:function:: PyObject* PySequence_InPlaceConcat(PyObject *o1, PyObject *o2) - Return the concatenation of *o1* and *o2* on success, and *NULL* on failure. + Return the concatenation of *o1* and *o2* on success, and ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python expression ``o1 += o2``. .. c:function:: PyObject* PySequence_InPlaceRepeat(PyObject *o, Py_ssize_t count) - Return the result of repeating sequence object *o* *count* times, or *NULL* on + Return the result of repeating sequence object *o* *count* times, or ``NULL`` on failure. The operation is done *in-place* when *o* supports it. This is the equivalent of the Python expression ``o *= count``. .. c:function:: PyObject* PySequence_GetItem(PyObject *o, Py_ssize_t i) - Return the *i*\ th element of *o*, or *NULL* on failure. This is the equivalent of + Return the *i*\ th element of *o*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o[i]``. .. c:function:: PyObject* PySequence_GetSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2) - Return the slice of sequence object *o* between *i1* and *i2*, or *NULL* on + Return the slice of sequence object *o* between *i1* and *i2*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o[i1:i2]``. @@ -69,7 +69,7 @@ Sequence Protocol is the equivalent of the Python statement ``o[i] = v``. This function *does not* steal a reference to *v*. - If *v* is *NULL*, the element is deleted, however this feature is + If *v* is ``NULL``, the element is deleted, however this feature is deprecated in favour of using :c:func:`PySequence_DelItem`. @@ -114,7 +114,7 @@ Sequence Protocol .. c:function:: PyObject* PySequence_List(PyObject *o) Return a list object with the same contents as the sequence or iterable *o*, - or *NULL* on failure. The returned list is guaranteed to be new. This is + or ``NULL`` on failure. The returned list is guaranteed to be new. This is equivalent to the Python expression ``list(o)``. @@ -123,38 +123,45 @@ Sequence Protocol .. index:: builtin: tuple Return a tuple object with the same contents as the sequence or iterable *o*, - or *NULL* on failure. If *o* is a tuple, a new reference will be returned, + or ``NULL`` on failure. If *o* is a tuple, a new reference will be returned, otherwise a tuple will be constructed with the appropriate contents. This is equivalent to the Python expression ``tuple(o)``. .. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m) - Return the sequence or iterable *o* as a list, unless it is already a tuple or list, in - which case *o* is returned. Use :c:func:`PySequence_Fast_GET_ITEM` to access - the members of the result. Returns *NULL* on failure. If the object is not - a sequence or iterable, raises :exc:`TypeError` with *m* as the message text. + Return the sequence or iterable *o* as an object usable by the other + ``PySequence_Fast*`` family of functions. If the object is not a sequence or + iterable, raises :exc:`TypeError` with *m* as the message text. Returns + ``NULL`` on failure. + + The ``PySequence_Fast*`` functions are thus named because they assume + *o* is a :c:type:`PyTupleObject` or a :c:type:`PyListObject` and access + the data fields of *o* directly. + + As a CPython implementation detail, if *o* is already a sequence or list, it + will be returned. .. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o) Returns the length of *o*, assuming that *o* was returned by - :c:func:`PySequence_Fast` and that *o* is not *NULL*. The size can also be + :c:func:`PySequence_Fast` and that *o* is not ``NULL``. The size can also be gotten by calling :c:func:`PySequence_Size` on *o*, but - :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list - or tuple. + :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a + list or tuple. .. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i) Return the *i*\ th element of *o*, assuming that *o* was returned by - :c:func:`PySequence_Fast`, *o* is not *NULL*, and that *i* is within bounds. + :c:func:`PySequence_Fast`, *o* is not ``NULL``, and that *i* is within bounds. .. c:function:: PyObject** PySequence_Fast_ITEMS(PyObject *o) Return the underlying array of PyObject pointers. Assumes that *o* was returned - by :c:func:`PySequence_Fast` and *o* is not *NULL*. + by :c:func:`PySequence_Fast` and *o* is not ``NULL``. Note, if a list gets resized, the reallocation may relocate the items array. So, only use the underlying array pointer in contexts where the sequence @@ -163,7 +170,7 @@ Sequence Protocol .. c:function:: PyObject* PySequence_ITEM(PyObject *o, Py_ssize_t i) - Return the *i*\ th element of *o* or *NULL* on failure. Macro form of + Return the *i*\ th element of *o* or ``NULL`` on failure. Faster form of :c:func:`PySequence_GetItem` but without checking that :c:func:`PySequence_Check` on *o* is true and without adjustment for negative indices. diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 64b6dde3b7482e..879f394d966cd0 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _setobjects: @@ -80,8 +80,8 @@ the constructor functions work with any iterable Python object. .. c:function:: PyObject* PySet_New(PyObject *iterable) Return a new :class:`set` containing objects returned by the *iterable*. The - *iterable* may be *NULL* to create a new empty set. Return the new set on - success or *NULL* on failure. Raise :exc:`TypeError` if *iterable* is not + *iterable* may be ``NULL`` to create a new empty set. Return the new set on + success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is not actually iterable. The constructor is also useful for copying a set (``c=set(s)``). @@ -89,8 +89,8 @@ the constructor functions work with any iterable Python object. .. c:function:: PyObject* PyFrozenSet_New(PyObject *iterable) Return a new :class:`frozenset` containing objects returned by the *iterable*. - The *iterable* may be *NULL* to create a new empty frozenset. Return the new - set on success or *NULL* on failure. Raise :exc:`TypeError` if *iterable* is + The *iterable* may be ``NULL`` to create a new empty frozenset. Return the new + set on success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is not actually iterable. @@ -149,7 +149,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. .. c:function:: PyObject* PySet_Pop(PyObject *set) Return a new reference to an arbitrary object in the *set*, and removes the - object from the *set*. Return *NULL* on failure. Raise :exc:`KeyError` if the + object from the *set*. Return ``NULL`` on failure. Raise :exc:`KeyError` if the set is empty. Raise a :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. @@ -157,10 +157,3 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. .. c:function:: int PySet_Clear(PyObject *set) Empty an existing set of all elements. - - -.. c:function:: int PySet_ClearFreeList() - - Clear the free list. Return the total number of freed items. - - .. versionadded:: 3.3 diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 8ad9a29b256ebc..48a58c6c6f7e36 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _slice-objects: @@ -14,15 +14,15 @@ Slice Objects .. c:function:: int PySlice_Check(PyObject *ob) - Return true if *ob* is a slice object; *ob* must not be *NULL*. + Return true if *ob* is a slice object; *ob* must not be ``NULL``. .. c:function:: PyObject* PySlice_New(PyObject *start, PyObject *stop, PyObject *step) Return a new slice object with the given values. The *start*, *stop*, and *step* parameters are used as the values of the slice object attributes of - the same names. Any of the values may be *NULL*, in which case the - ``None`` will be used for the corresponding attribute. Return *NULL* if + the same names. Any of the values may be ``NULL``, in which case the + ``None`` will be used for the corresponding attribute. Return ``NULL`` if the new object could not be allocated. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 5b771dd4adcefb..9c05cb3c82dfbe 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _stable: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index da45da1d3c70db..a9e1c6fbcc3f94 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _common-structs: @@ -9,6 +9,10 @@ There are a large number of structures which are used in the definition of object types for Python. This section describes these structures and how they are used. + +Base object types and macros +---------------------------- + All Python objects ultimately share a small number of fields at the beginning of the object's representation in memory. These are represented by the :c:type:`PyObject` and :c:type:`PyVarObject` types, which are defined, in turn, @@ -58,29 +62,62 @@ the definition of all other Python objects. See documentation of :c:type:`PyVarObject` above. -.. c:macro:: Py_TYPE(o) +.. c:function:: PyTypeObject* Py_TYPE(const PyObject *o) + + Get the type of the Python object *o*. + + Return a borrowed reference. + + .. versionchanged:: 3.10 + :c:func:`Py_TYPE()` is changed to the inline static function. + Use :c:func:`Py_SET_TYPE()` to set an object type. + + +.. c:function:: int Py_IS_TYPE(PyObject *o, PyTypeObject *type) + + Return non-zero if the object *o* type is *type*. Return zero otherwise. + Equivalent to: ``Py_TYPE(o) == type``. + + .. versionadded:: 3.9 + + +.. c:function:: void Py_SET_TYPE(PyObject *o, PyTypeObject *type) + + Set the object *o* type to *type*. + + .. versionadded:: 3.9 - This macro is used to access the :attr:`ob_type` member of a Python object. - It expands to:: - (((PyObject*)(o))->ob_type) +.. c:function:: Py_ssize_t Py_REFCNT(const PyObject *o) + Get the reference count of the Python object *o*. -.. c:macro:: Py_REFCNT(o) + .. versionchanged:: 3.10 + :c:func:`Py_REFCNT()` is changed to the inline static function. + Use :c:func:`Py_SET_REFCNT()` to set an object reference count. - This macro is used to access the :attr:`ob_refcnt` member of a Python - object. - It expands to:: - (((PyObject*)(o))->ob_refcnt) +.. c:function:: void Py_SET_REFCNT(PyObject *o, Py_ssize_t refcnt) + Set the object *o* reference counter to *refcnt*. -.. c:macro:: Py_SIZE(o) + .. versionadded:: 3.9 - This macro is used to access the :attr:`ob_size` member of a Python object. - It expands to:: - (((PyVarObject*)(o))->ob_size) +.. c:function:: Py_ssize_t Py_SIZE(const PyVarObject *o) + + Get the size of the Python object *o*. + + .. versionchanged:: 3.10 + :c:func:`Py_SIZE()` is changed to the inline static function. + Use :c:func:`Py_SET_SIZE()` to set an object size. + + +.. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size) + + Set the object *o* size to *size*. + + .. versionadded:: 3.9 .. c:macro:: PyObject_HEAD_INIT(type) @@ -102,22 +139,68 @@ the definition of all other Python objects. 1, type, size, +Implementing functions and methods +---------------------------------- + .. c:type:: PyCFunction Type of the functions used to implement most Python callables in C. - Functions of this type take two :c:type:`PyObject\*` parameters and return - one such value. If the return value is *NULL*, an exception shall have - been set. If not *NULL*, the return value is interpreted as the return + Functions of this type take two :c:type:`PyObject*` parameters and return + one such value. If the return value is ``NULL``, an exception shall have + been set. If not ``NULL``, the return value is interpreted as the return value of the function as exposed in Python. The function must return a new reference. + The function signature is:: + + PyObject *PyCFunction(PyObject *self, + PyObject *args); .. c:type:: PyCFunctionWithKeywords - Type of the functions used to implement Python callables in C that take - keyword arguments: they take three :c:type:`PyObject\*` parameters and return - one such value. See :c:type:`PyCFunction` above for the meaning of the return - value. + Type of the functions used to implement Python callables in C + with signature :const:`METH_VARARGS | METH_KEYWORDS`. + The function signature is:: + + PyObject *PyCFunctionWithKeywords(PyObject *self, + PyObject *args, + PyObject *kwargs); + + +.. c:type:: _PyCFunctionFast + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL`. + The function signature is:: + + PyObject *_PyCFunctionFast(PyObject *self, + PyObject *const *args, + Py_ssize_t nargs); + +.. c:type:: _PyCFunctionFastWithKeywords + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL | METH_KEYWORDS`. + The function signature is:: + + PyObject *_PyCFunctionFastWithKeywords(PyObject *self, + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames); + +.. c:type:: PyCMethod + + Type of the functions used to implement Python callables in C + with signature :const:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS`. + The function signature is:: + + PyObject *PyCMethod(PyObject *self, + PyTypeObject *defining_class, + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames) + + .. versionadded:: 3.9 .. c:type:: PyMethodDef @@ -141,36 +224,78 @@ the definition of all other Python objects. +------------------+---------------+-------------------------------+ The :attr:`ml_meth` is a C function pointer. The functions may be of different -types, but they always return :c:type:`PyObject\*`. If the function is not of +types, but they always return :c:type:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as -:c:type:`PyObject\*`, it is common that the method implementation uses the +:c:type:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. The :attr:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding -convention. Of the calling convention flags, only :const:`METH_VARARGS` and -:const:`METH_KEYWORDS` can be combined. Any of the calling convention flags -can be combined with a binding flag. +convention. +There are these calling conventions: .. data:: METH_VARARGS This is the typical calling convention, where the methods have the type - :c:type:`PyCFunction`. The function expects two :c:type:`PyObject\*` values. + :c:type:`PyCFunction`. The function expects two :c:type:`PyObject*` values. The first one is the *self* object for methods; for module functions, it is the module object. The second parameter (often called *args*) is a tuple object representing all arguments. This parameter is typically processed using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`. -.. data:: METH_KEYWORDS +.. data:: METH_VARARGS | METH_KEYWORDS Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`. - The function expects three parameters: *self*, *args*, and a dictionary of - all the keyword arguments. The flag must be combined with - :const:`METH_VARARGS`, and the parameters are typically processed using - :c:func:`PyArg_ParseTupleAndKeywords`. + The function expects three parameters: *self*, *args*, *kwargs* where + *kwargs* is a dictionary of all the keyword arguments or possibly ``NULL`` + if there are no keyword arguments. The parameters are typically processed + using :c:func:`PyArg_ParseTupleAndKeywords`. + + +.. data:: METH_FASTCALL + + Fast calling convention supporting only positional arguments. + The methods have the type :c:type:`_PyCFunctionFast`. + The first parameter is *self*, the second parameter is a C array + of :c:type:`PyObject*` values indicating the arguments and the third + parameter is the number of arguments (the length of the array). + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 + + +.. data:: METH_FASTCALL | METH_KEYWORDS + + Extension of :const:`METH_FASTCALL` supporting also keyword arguments, + with methods of type :c:type:`_PyCFunctionFastWithKeywords`. + Keyword arguments are passed the same way as in the + :ref:`vectorcall protocol `: + there is an additional fourth :c:type:`PyObject*` parameter + which is a tuple representing the names of the keyword arguments + (which are guaranteed to be strings) + or possibly ``NULL`` if there are no keywords. The values of the keyword + arguments are stored in the *args* array, after the positional arguments. + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 + + +.. data:: METH_METHOD | METH_FASTCALL | METH_KEYWORDS + + Extension of :const:`METH_FASTCALL | METH_KEYWORDS` supporting the *defining + class*, that is, the class that contains the method in question. + The defining class might be a superclass of ``Py_TYPE(self)``. + + The method needs to be of type :c:type:`PyCMethod`, the same as for + ``METH_FASTCALL | METH_KEYWORDS`` with ``defining_class`` argument added after + ``self``. + + .. versionadded:: 3.9 .. data:: METH_NOARGS @@ -179,7 +304,7 @@ can be combined with a binding flag. they are listed with the :const:`METH_NOARGS` flag. They need to be of type :c:type:`PyCFunction`. The first parameter is typically named *self* and will hold a reference to the module or object instance. In all cases the second - parameter will be *NULL*. + parameter will be ``NULL``. .. data:: METH_O @@ -187,7 +312,7 @@ can be combined with a binding flag. Methods with a single object argument can be listed with the :const:`METH_O` flag, instead of invoking :c:func:`PyArg_ParseTuple` with a ``"O"`` argument. They have the type :c:type:`PyCFunction`, with the *self* parameter, and a - :c:type:`PyObject\*` parameter representing the single argument. + :c:type:`PyObject*` parameter representing the single argument. These two constants are not used to indicate the calling convention but the @@ -210,7 +335,7 @@ method. .. index:: builtin: staticmethod - The method will be passed *NULL* as the first parameter rather than an + The method will be passed ``NULL`` as the first parameter rather than an instance of the type. This is used to create *static methods*, similar to what is created when using the :func:`staticmethod` built-in function. @@ -231,6 +356,9 @@ definition with the same method name. than wrapper object calls. +Accessing attributes of extension types +--------------------------------------- + .. c:type:: PyMemberDef Structure which describes an attribute of a type which corresponds to a C @@ -284,7 +412,7 @@ definition with the same method name. =============== ================== :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` differ in that - :c:macro:`T_OBJECT` returns ``None`` if the member is *NULL* and + :c:macro:`T_OBJECT` returns ``None`` if the member is ``NULL`` and :c:macro:`T_OBJECT_EX` raises an :exc:`AttributeError`. Try to use :c:macro:`T_OBJECT_EX` over :c:macro:`T_OBJECT` because :c:macro:`T_OBJECT_EX` handles use of the :keyword:`del` statement on that attribute more correctly @@ -294,8 +422,23 @@ definition with the same method name. read-only access. Using :c:macro:`T_STRING` for :attr:`type` implies :c:macro:`READONLY`. :c:macro:`T_STRING` data is interpreted as UTF-8. Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` - members can be deleted. (They are set to *NULL*). + members can be deleted. (They are set to ``NULL``). + + .. _pymemberdef-offsets: + + Heap allocated types (created using :c:func:`PyType_FromSpec` or similar), + ``PyMemberDef`` may contain definitions for the special members + ``__dictoffset__``, ``__weaklistoffset__`` and ``__vectorcalloffset__``, + corresponding to + :c:member:`~PyTypeObject.tp_dictoffset`, + :c:member:`~PyTypeObject.tp_weaklistoffset` and + :c:member:`~PyTypeObject.tp_vectorcall_offset` in type objects. + These must be defined with ``T_PYSSIZET`` and ``READONLY``, for example:: + static PyMemberDef spam_type_members[] = { + {"__dictoffset__", T_PYSSIZET, offsetof(Spam_object, dict), READONLY}, + {NULL} /* Sentinel */ + }; .. c:type:: PyGetSetDef @@ -320,18 +463,18 @@ definition with the same method name. | | | getter and setter | +-------------+------------------+-----------------------------------+ - The ``get`` function takes one :c:type:`PyObject\*` parameter (the + The ``get`` function takes one :c:type:`PyObject*` parameter (the instance) and a function pointer (the associated ``closure``):: typedef PyObject *(*getter)(PyObject *, void *); - It should return a new reference on success or *NULL* with a set exception + It should return a new reference on success or ``NULL`` with a set exception on failure. - ``set`` functions take two :c:type:`PyObject\*` parameters (the instance and + ``set`` functions take two :c:type:`PyObject*` parameters (the instance and the value to be set) and a function pointer (the associated ``closure``):: typedef int (*setter)(PyObject *, PyObject *, void *); - In case the attribute should be deleted the second parameter is *NULL*. + In case the attribute should be deleted the second parameter is ``NULL``. Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index c8ea78363dfa51..9ac91790978926 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _os: @@ -22,7 +22,7 @@ Operating System Utilities Return true (nonzero) if the standard I/O file *fp* with name *filename* is deemed interactive. This is the case for files for which ``isatty(fileno(fp))`` is true. If the global flag :c:data:`Py_InteractiveFlag` is true, this function - also returns true if the *filename* pointer is *NULL* or if the name is equal to + also returns true if the *filename* pointer is ``NULL`` or if the name is equal to one of the strings ``''`` or ``'???'``. @@ -33,6 +33,12 @@ Operating System Utilities that clones the current process. Only available on systems where :c:func:`fork` is defined. + .. warning:: + The C :c:func:`fork` call should only be made from the + :ref:`"main" thread ` (of the + :ref:`"main" interpreter `). The same is + true for ``PyOS_BeforeFork()``. + .. versionadded:: 3.7 @@ -44,6 +50,12 @@ Operating System Utilities of whether process cloning was successful. Only available on systems where :c:func:`fork` is defined. + .. warning:: + The C :c:func:`fork` call should only be made from the + :ref:`"main" thread ` (of the + :ref:`"main" interpreter `). The same is + true for ``PyOS_AfterFork_Parent()``. + .. versionadded:: 3.7 @@ -55,6 +67,12 @@ Operating System Utilities any chance the process will call back into the Python interpreter. Only available on systems where :c:func:`fork` is defined. + .. warning:: + The C :c:func:`fork` call should only be made from the + :ref:`"main" thread ` (of the + :ref:`"main" interpreter `). The same is + true for ``PyOS_AfterFork_Child()``. + .. versionadded:: 3.7 .. seealso:: @@ -201,12 +219,12 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: PyObject *PySys_GetObject(const char *name) - Return the object *name* from the :mod:`sys` module or *NULL* if it does + Return the object *name* from the :mod:`sys` module or ``NULL`` if it does not exist, without setting an exception. .. c:function:: int PySys_SetObject(const char *name, PyObject *v) - Set *name* in the :mod:`sys` module to *v* unless *v* is *NULL*, in which + Set *name* in the :mod:`sys` module to *v* unless *v* is ``NULL``, in which case *name* is deleted from the sys module. Returns ``0`` on success, ``-1`` on error. @@ -283,12 +301,76 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: PyObject *PySys_GetXOptions() Return the current dictionary of :option:`-X` options, similarly to - :data:`sys._xoptions`. On error, *NULL* is returned and an exception is + :data:`sys._xoptions`. On error, ``NULL`` is returned and an exception is set. .. versionadded:: 3.2 +.. c:function:: int PySys_Audit(const char *event, const char *format, ...) + + Raise an auditing event with any active hooks. Return zero for success + and non-zero with an exception set on failure. + + If any hooks have been added, *format* and other arguments will be used + to construct a tuple to pass. Apart from ``N``, the same format characters + as used in :c:func:`Py_BuildValue` are available. If the built value is not + a tuple, it will be added into a single-element tuple. (The ``N`` format + option consumes a reference, but since there is no way to know whether + arguments to this function will be consumed, using it may cause reference + leaks.) + + Note that ``#`` format characters should always be treated as + ``Py_ssize_t``, regardless of whether ``PY_SSIZE_T_CLEAN`` was defined. + + :func:`sys.audit` performs the same function from Python code. + + .. versionadded:: 3.8 + + .. versionchanged:: 3.8.2 + + Require ``Py_ssize_t`` for ``#`` format characters. Previously, an + unavoidable deprecation warning was raised. + + +.. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) + + Append the callable *hook* to the list of active auditing hooks. + Return zero for success + and non-zero on failure. If the runtime has been initialized, also set an + error on failure. Hooks added through this API are called for all + interpreters created by the runtime. + + The *userData* pointer is passed into the hook function. Since hook + functions may be called from different runtimes, this pointer should not + refer directly to Python state. + + This function is safe to call before :c:func:`Py_Initialize`. When called + after runtime initialization, existing audit hooks are notified and may + silently abort the operation by raising an error subclassed from + :class:`Exception` (other errors will not be silenced). + + The hook function is of type :c:type:`int (*)(const char *event, PyObject + *args, void *userData)`, where *args* is guaranteed to be a + :c:type:`PyTupleObject`. The hook function is always called with the GIL + held by the Python interpreter that raised the event. + + See :pep:`578` for a detailed description of auditing. Functions in the + runtime and standard library that raise events are listed in the + :ref:`audit events table `. + Details are in each function's documentation. + + .. audit-event:: sys.addaudithook "" c.PySys_AddAuditHook + + If the interpreter is initialized, this function raises a auditing event + ``sys.addaudithook`` with no arguments. If any existing hooks raise an + exception derived from :class:`Exception`, the new hook will not be + added and the exception is cleared. As a result, callers cannot assume + that their hook has been added unless they control all existing hooks. + + .. versionadded:: 3.8 + + .. _processcontrol: Process Control @@ -306,6 +388,13 @@ Process Control function :c:func:`abort` is called which will attempt to produce a :file:`core` file. + The ``Py_FatalError()`` function is replaced with a macro which logs + automatically the name of the current function, unless the + ``Py_LIMITED_API`` macro is defined. + + .. versionchanged:: 3.9 + Log the function name automatically. + .. c:function:: void Py_Exit(int status) diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 20bf9f0b0804cd..bf751e44acde09 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _tupleobjects: @@ -33,12 +33,12 @@ Tuple Objects .. c:function:: PyObject* PyTuple_New(Py_ssize_t len) - Return a new tuple object of size *len*, or *NULL* on failure. + Return a new tuple object of size *len*, or ``NULL`` on failure. .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) - Return a new tuple object of size *n*, or *NULL* on failure. The tuple values + Return a new tuple object of size *n*, or ``NULL`` on failure. The tuple values are initialized to the subsequent *n* C arguments pointing to Python objects. ``PyTuple_Pack(2, a, b)`` is equivalent to ``Py_BuildValue("(OO)", a, b)``. @@ -50,14 +50,14 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) - Return the size of the tuple *p*, which must be non-*NULL* and point to a tuple; + Return the size of the tuple *p*, which must be non-``NULL`` and point to a tuple; no error checking is performed. .. c:function:: PyObject* PyTuple_GetItem(PyObject *p, Py_ssize_t pos) Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is - out of bounds, return *NULL* and sets an :exc:`IndexError` exception. + out of bounds, return ``NULL`` and set an :exc:`IndexError` exception. .. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) @@ -67,18 +67,21 @@ Tuple Objects .. c:function:: PyObject* PyTuple_GetSlice(PyObject *p, Py_ssize_t low, Py_ssize_t high) - Take a slice of the tuple pointed to by *p* from *low* to *high* and return it - as a new tuple. + Return the slice of the tuple pointed to by *p* between *low* and *high*, + or ``NULL`` on failure. This is the equivalent of the Python expression + ``p[low:high]``. Indexing from the end of the list is not supported. .. c:function:: int PyTuple_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) Insert a reference to object *o* at position *pos* of the tuple pointed to by - *p*. Return ``0`` on success. + *p*. Return ``0`` on success. If *pos* is out of bounds, return ``-1`` + and set an :exc:`IndexError` exception. .. note:: - This function "steals" a reference to *o*. + This function "steals" a reference to *o* and discards a reference to + an item already in the tuple at the affected position. .. c:function:: void PyTuple_SET_ITEM(PyObject *p, Py_ssize_t pos, PyObject *o) @@ -88,7 +91,10 @@ Tuple Objects .. note:: - This function "steals" a reference to *o*. + This macro "steals" a reference to *o*, and, unlike + :c:func:`PyTuple_SetItem`, does *not* discard a reference to any item that + is being replaced; any reference in the tuple at position *pos* will be + leaked. .. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) @@ -101,15 +107,10 @@ Tuple Objects only more efficiently. Returns ``0`` on success. Client code should never assume that the resulting value of ``*p`` will be the same as before calling this function. If the object referenced by ``*p`` is replaced, the original - ``*p`` is destroyed. On failure, returns ``-1`` and sets ``*p`` to *NULL*, and + ``*p`` is destroyed. On failure, returns ``-1`` and sets ``*p`` to ``NULL``, and raises :exc:`MemoryError` or :exc:`SystemError`. -.. c:function:: int PyTuple_ClearFreeList() - - Clear the free list. Return the total number of freed items. - - Struct Sequence Objects ----------------------- @@ -141,45 +142,48 @@ type. Contains the meta information of a struct sequence type to create. - +-------------------+------------------------------+------------------------------------+ - | Field | C Type | Meaning | - +===================+==============================+====================================+ - | ``name`` | ``const char *`` | name of the struct sequence type | - +-------------------+------------------------------+------------------------------------+ - | ``doc`` | ``const char *`` | pointer to docstring for the type | - | | | or NULL to omit | - +-------------------+------------------------------+------------------------------------+ - | ``fields`` | ``PyStructSequence_Field *`` | pointer to *NULL*-terminated array | - | | | with field names of the new type | - +-------------------+------------------------------+------------------------------------+ - | ``n_in_sequence`` | ``int`` | number of fields visible to the | - | | | Python side (if used as tuple) | - +-------------------+------------------------------+------------------------------------+ + +-------------------+------------------------------+--------------------------------------+ + | Field | C Type | Meaning | + +===================+==============================+======================================+ + | ``name`` | ``const char *`` | name of the struct sequence type | + +-------------------+------------------------------+--------------------------------------+ + | ``doc`` | ``const char *`` | pointer to docstring for the type | + | | | or ``NULL`` to omit | + +-------------------+------------------------------+--------------------------------------+ + | ``fields`` | ``PyStructSequence_Field *`` | pointer to ``NULL``-terminated array | + | | | with field names of the new type | + +-------------------+------------------------------+--------------------------------------+ + | ``n_in_sequence`` | ``int`` | number of fields visible to the | + | | | Python side (if used as tuple) | + +-------------------+------------------------------+--------------------------------------+ .. c:type:: PyStructSequence_Field Describes a field of a struct sequence. As a struct sequence is modeled as a - tuple, all fields are typed as :c:type:`PyObject\*`. The index in the + tuple, all fields are typed as :c:type:`PyObject*`. The index in the :attr:`fields` array of the :c:type:`PyStructSequence_Desc` determines which field of the struct sequence is described. - +-----------+------------------+--------------------------------------+ - | Field | C Type | Meaning | - +===========+==================+======================================+ - | ``name`` | ``const char *`` | name for the field or *NULL* to end | - | | | the list of named fields, set to | - | | | PyStructSequence_UnnamedField to | - | | | leave unnamed | - +-----------+------------------+--------------------------------------+ - | ``doc`` | ``const char *`` | field docstring or *NULL* to omit | - +-----------+------------------+--------------------------------------+ + +-----------+------------------+-----------------------------------------+ + | Field | C Type | Meaning | + +===========+==================+=========================================+ + | ``name`` | ``const char *`` | name for the field or ``NULL`` to end | + | | | the list of named fields, set to | + | | | :c:data:`PyStructSequence_UnnamedField` | + | | | to leave unnamed | + +-----------+------------------+-----------------------------------------+ + | ``doc`` | ``const char *`` | field docstring or ``NULL`` to omit | + +-----------+------------------+-----------------------------------------+ -.. c:var:: char* PyStructSequence_UnnamedField +.. c:var:: const char * const PyStructSequence_UnnamedField Special value for a field name to leave it unnamed. + .. versionchanged:: 3.9 + The type was changed from ``char *``. + .. c:function:: PyObject* PyStructSequence_New(PyTypeObject *type) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 4dfd53fb9f076f..73f26875d8194a 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _typeobjects: @@ -21,14 +21,14 @@ Type Objects .. c:function:: int PyType_Check(PyObject *o) - Return true if the object *o* is a type object, including instances of types - derived from the standard type object. Return false in all other cases. + Return non-zero if the object *o* is a type object, including instances of + types derived from the standard type object. Return 0 in all other cases. .. c:function:: int PyType_CheckExact(PyObject *o) - Return true if the object *o* is a type object, but not a subtype of the - standard type object. Return false in all other cases. + Return non-zero if the object *o* is a type object, but not a subtype of the + standard type object. Return 0 in all other cases. .. c:function:: unsigned int PyType_ClearCache() @@ -57,8 +57,8 @@ Type Objects .. c:function:: int PyType_HasFeature(PyTypeObject *o, int feature) - Return true if the type object *o* sets the feature *feature*. Type features - are denoted by single bit flags. + Return non-zero if the type object *o* sets the feature *feature*. + Type features are denoted by single bit flags. .. c:function:: int PyType_IS_GC(PyTypeObject *o) @@ -81,7 +81,7 @@ Type Objects Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type object. Use Python's default memory allocation mechanism to allocate a new instance and - initialize all its contents to *NULL*. + initialize all its contents to ``NULL``. .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -95,24 +95,169 @@ Type Objects from a type's base class. Return ``0`` on success, or return ``-1`` and sets an exception on error. -.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) +.. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) + + Return the function pointer stored in the given slot. If the + result is ``NULL``, this indicates that either the slot is ``NULL``, + or that the function was called with invalid parameters. + Callers will typically cast the result pointer into the appropriate + function type. + + See :c:member:`PyType_Slot.slot` for possible values of the *slot* argument. + + An exception is raised if *type* is not a heap type. + + .. versionadded:: 3.4 + +.. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) + + Return the module object associated with the given type when the type was + created using :c:func:`PyType_FromModuleAndSpec`. + + If no module is associated with the given type, sets :py:class:`TypeError` + and returns ``NULL``. + + This function is usually used to get the module in which a method is defined. + Note that in such a method, ``PyType_GetModule(Py_TYPE(self))`` + may not return the intended result. + ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses + are not necessarily defined in the same module as their superclass. + See :c:type:`PyCMethod` to get the class that defines the method. + + .. versionadded:: 3.9 - Creates and returns a heap type object from the *spec* passed to the function. +.. c:function:: void* PyType_GetModuleState(PyTypeObject *type) + + Return the state of the module object associated with the given type. + This is a shortcut for calling :c:func:`PyModule_GetState()` on the result + of :c:func:`PyType_GetModule`. + + If no module is associated with the given type, sets :py:class:`TypeError` + and returns ``NULL``. + + If the *type* has an associated module but its state is ``NULL``, + returns ``NULL`` without setting an exception. + + .. versionadded:: 3.9 + + +Creating Heap-Allocated Types +............................. + +The following functions and structs are used to create +:ref:`heap types `. + +.. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) + + Creates and returns a heap type object from the *spec* + (:const:`Py_TPFLAGS_HEAPTYPE`). + + If *bases* is a tuple, the created heap type contains all types contained + in it as base types. + + If *bases* is ``NULL``, the *Py_tp_base* slot is used instead. + If that also is ``NULL``, the new type derives from :class:`object`. + + The *module* argument can be used to record the module in which the new + class is defined. It must be a module object or ``NULL``. + If not ``NULL``, the module is associated with the new type and can later be + retreived with :c:func:`PyType_GetModule`. + The associated module is not inherited by subclasses; it must be specified + for each class individually. + + This function calls :c:func:`PyType_Ready` on the new type. + + .. versionadded:: 3.9 .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) - Creates and returns a heap type object from the *spec*. In addition to that, - the created heap type contains all types contained by the *bases* tuple as base - types. This allows the caller to reference other heap types as base types. + Equivalent to ``PyType_FromModuleAndSpec(NULL, spec, bases)``. .. versionadded:: 3.3 -.. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) +.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) - Return the function pointer stored in the given slot. If the - result is *NULL*, this indicates that either the slot is *NULL*, - or that the function was called with invalid parameters. - Callers will typically cast the result pointer into the appropriate - function type. + Equivalent to ``PyType_FromSpecWithBases(spec, NULL)``. - .. versionadded:: 3.4 +.. c:type:: PyType_Spec + + Structure defining a type's behavior. + + .. c:member:: const char* PyType_Spec.name + + Name of the type, used to set :c:member:`PyTypeObject.tp_name`. + + .. c:member:: int PyType_Spec.basicsize + .. c:member:: int PyType_Spec.itemsize + + Size of the instance in bytes, used to set + :c:member:`PyTypeObject.tp_basicsize` and + :c:member:`PyTypeObject.tp_itemsize`. + + .. c:member:: int PyType_Spec.flags + + Type flags, used to set :c:member:`PyTypeObject.tp_flags`. + + If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, + :c:func:`PyType_FromSpecWithBases` sets it automatically. + + .. c:member:: PyType_Slot *PyType_Spec.slots + + Array of :c:type:`PyType_Slot` structures. + Terminated by the special slot value ``{0, NULL}``. + +.. c:type:: PyType_Slot + + Structure defining optional functionality of a type, containing a slot ID + and a value pointer. + + .. c:member:: int PyType_Slot.slot + + A slot ID. + + Slot IDs are named like the field names of the structures + :c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, + :c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and + :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. + For example, use: + + * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` + * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` + * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + + The following fields cannot be set at all using :c:type:`PyType_Spec` and + :c:type:`PyType_Slot`: + + * :c:member:`~PyTypeObject.tp_dict` + * :c:member:`~PyTypeObject.tp_mro` + * :c:member:`~PyTypeObject.tp_cache` + * :c:member:`~PyTypeObject.tp_subclasses` + * :c:member:`~PyTypeObject.tp_weaklist` + * :c:member:`~PyTypeObject.tp_vectorcall` + * :c:member:`~PyTypeObject.tp_weaklistoffset` + (see :ref:`PyMemberDef `) + * :c:member:`~PyTypeObject.tp_dictoffset` + (see :ref:`PyMemberDef `) + * :c:member:`~PyTypeObject.tp_vectorcall_offset` + (see :ref:`PyMemberDef `) + + The following fields cannot be set using :c:type:`PyType_Spec` and + :c:type:`PyType_Slot` under the limited API: + + * :c:member:`~PyBufferProcs.bf_getbuffer` + * :c:member:`~PyBufferProcs.bf_releasebuffer` + + Setting :c:data:`Py_tp_bases` may be problematic on some platforms. + To avoid issues, use the *bases* argument of + :py:func:`PyType_FromSpecWithBases` instead. + + .. versionchanged:: 3.9 + + Slots in :c:type:`PyBufferProcs` in may be set in the unlimited API. + + .. c:member:: void *PyType_Slot.pfunc + + The desired value of the slot. In most cases, this is a pointer + to a function. + + May not be ``NULL``. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 0647a493303d26..ddcb8ae3d0950c 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _type-structs: @@ -20,7 +20,7 @@ functionality. The fields of the type object are examined in detail in this section. The fields will be described in the order in which they occur in the structure. -In addition to the following quick reference, the :ref:`examples` +In addition to the following quick reference, the :ref:`typedef-examples` section provides at-a-glance insight into the meaning and use of :c:type:`PyTypeObject`. @@ -36,143 +36,136 @@ Quick Reference .. table:: :widths: 18,18,18,1,1,1,1 - +---------------------------------------------+-----------------------------------+-------------------+---------------+ - | PyTypeObject Slot [#slots]_ | :ref:`Type ` | special | Info [#cols]_ | - | | | methods/attrs +---+---+---+---+ - | | | | O | T | D | I | - +=============================================+===================================+===================+===+===+===+===+ - | :c:member:`~PyTypeObject.tp_name` | const char * | __name__ | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_basicsize` | Py_ssize_t | | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_itemsize` | Py_ssize_t | | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dealloc` | :c:type:`destructor` | | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_print`) | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_getattr`) | :c:type:`getattrfunc` | __getattribute__, | | | | G | - | | | __getattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_setattr`) | :c:type:`setattrfunc` | __setattr__, | | | | G | - | | | __delattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_async` | :c:type:`PyAsyncMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_repr` | :c:type:`reprfunc` | __repr__ | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_number` | :c:type:`PyNumberMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_sequence` | :c:type:`PySequenceMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_mapping` | :c:type:`PyMappingMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_hash` | :c:type:`hashfunc` | __hash__ | X | | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_call` | :c:type:`ternaryfunc` | __call__ | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_str` | :c:type:`reprfunc` | __str__ | X | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_getattro` | :c:type:`getattrofunc` | __getattribute__, | X | X | | G | - | | | __getattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | - | | | __delattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_doc` | const char * | __doc__ | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_traverse` | :c:type:`traverseproc` | | | X | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_clear` | :c:type:`inquiry` | | | X | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_richcompare` | :c:type:`richcmpfunc` | __lt__, | X | | | G | - | | | __le__, | | | | | - | | | __eq__, | | | | | - | | | __ne__, | | | | | - | | | __gt__, | | | | | - | | | __ge__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_weaklistoffset` | Py_ssize_t | | | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_iter` | :c:type:`getiterfunc` | __iter__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_iternext` | :c:type:`iternextfunc` | __next__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_methods` | :c:type:`PyMethodDef` [] | | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_members` | :c:type:`PyMemberDef` [] | | | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_getset` | :c:type:`PyGetSetDef` [] | | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_base` | :c:type:`PyTypeObject` * | __base__ | | | X | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dict` | :c:type:`PyObject` * | __dict__ | | | ? | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_descr_get` | :c:type:`descrgetfunc` | __get__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_descr_set` | :c:type:`descrsetfunc` | __set__, | | | | X | - | | | __delete__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dictoffset` | Py_ssize_t | | | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_init` | :c:type:`initproc` | __init__ | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_alloc` | :c:type:`allocfunc` | | X | | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_new` | :c:type:`newfunc` | __new__ | X | X | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_free` | :c:type:`freefunc` | | X | X | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_is_gc` | :c:type:`inquiry` | | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | <:c:member:`~PyTypeObject.tp_bases`> | :c:type:`PyObject` * | __bases__ | | | ~ | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | <:c:member:`~PyTypeObject.tp_mro`> | :c:type:`PyObject` * | __mro__ | | | ~ | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_subclasses`] | :c:type:`PyObject` * | __subclasses__ | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_weaklist`] | :c:type:`PyObject` * | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_del`) | :c:type:`destructor` | | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_version_tag`] | unsigned int | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_finalize` | :c:type:`destructor` | __del__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - -If :const:`COUNT_ALLOCS` is defined then the following (internal-only) -fields exist as well: - -* :c:member:`~PyTypeObject.tp_allocs` -* :c:member:`~PyTypeObject.tp_frees` -* :c:member:`~PyTypeObject.tp_maxalloc` -* :c:member:`~PyTypeObject.tp_prev` -* :c:member:`~PyTypeObject.tp_next` + +------------------------------------------------+-----------------------------------+-------------------+---------------+ + | PyTypeObject Slot [#slots]_ | :ref:`Type ` | special | Info [#cols]_ | + | | | methods/attrs +---+---+---+---+ + | | | | O | T | D | I | + +================================================+===================================+===================+===+===+===+===+ + | :c:member:`~PyTypeObject.tp_name` | const char * | __name__ | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_basicsize` | Py_ssize_t | | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_itemsize` | Py_ssize_t | | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dealloc` | :c:type:`destructor` | | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_vectorcall_offset` | Py_ssize_t | | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_getattr`) | :c:type:`getattrfunc` | __getattribute__, | | | | G | + | | | __getattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_setattr`) | :c:type:`setattrfunc` | __setattr__, | | | | G | + | | | __delattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_async` | :c:type:`PyAsyncMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_repr` | :c:type:`reprfunc` | __repr__ | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_number` | :c:type:`PyNumberMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_sequence` | :c:type:`PySequenceMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_mapping` | :c:type:`PyMappingMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_hash` | :c:type:`hashfunc` | __hash__ | X | | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_call` | :c:type:`ternaryfunc` | __call__ | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_str` | :c:type:`reprfunc` | __str__ | X | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_getattro` | :c:type:`getattrofunc` | __getattribute__, | X | X | | G | + | | | __getattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | + | | | __delattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_doc` | const char * | __doc__ | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_traverse` | :c:type:`traverseproc` | | | X | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_clear` | :c:type:`inquiry` | | | X | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_richcompare` | :c:type:`richcmpfunc` | __lt__, | X | | | G | + | | | __le__, | | | | | + | | | __eq__, | | | | | + | | | __ne__, | | | | | + | | | __gt__, | | | | | + | | | __ge__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_weaklistoffset` | Py_ssize_t | | | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_iter` | :c:type:`getiterfunc` | __iter__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_iternext` | :c:type:`iternextfunc` | __next__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_methods` | :c:type:`PyMethodDef` [] | | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_members` | :c:type:`PyMemberDef` [] | | | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_getset` | :c:type:`PyGetSetDef` [] | | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_base` | :c:type:`PyTypeObject` * | __base__ | | | X | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dict` | :c:type:`PyObject` * | __dict__ | | | ? | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_descr_get` | :c:type:`descrgetfunc` | __get__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_descr_set` | :c:type:`descrsetfunc` | __set__, | | | | X | + | | | __delete__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dictoffset` | Py_ssize_t | | | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_init` | :c:type:`initproc` | __init__ | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_alloc` | :c:type:`allocfunc` | | X | | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_new` | :c:type:`newfunc` | __new__ | X | X | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_free` | :c:type:`freefunc` | | X | X | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_is_gc` | :c:type:`inquiry` | | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | <:c:member:`~PyTypeObject.tp_bases`> | :c:type:`PyObject` * | __bases__ | | | ~ | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | <:c:member:`~PyTypeObject.tp_mro`> | :c:type:`PyObject` * | __mro__ | | | ~ | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_subclasses`] | :c:type:`PyObject` * | __subclasses__ | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_weaklist`] | :c:type:`PyObject` * | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_del`) | :c:type:`destructor` | | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_version_tag`] | unsigned int | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_finalize` | :c:type:`destructor` | __del__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] A slot name in parentheses indicates it is (effectively) deprecated. Names in angle brackets should be treated as read-only. Names in square brackets are for internal use only. - "" (as a prefix) means the field is required (must be non-*NULL*). + "" (as a prefix) means the field is required (must be non-``NULL``). .. [#cols] Columns: **"O"**: set on :c:type:`PyBaseObject_Type` **"T"**: set on :c:type:`PyType_Type` - **"D"**: default (if slot is set to *NULL*) + **"D"**: default (if slot is set to ``NULL``) .. code-block:: none - X - *PyType_Ready* sets this value if it is *NULL* - ~ - *PyType_Ready* always sets this value (it should be *NULL*) - ? - *PyType_Ready* may set this value depending on other slots + X - PyType_Ready sets this value if it is NULL + ~ - PyType_Ready always sets this value (it should be NULL) + ? - PyType_Ready may set this value depending on other slots Also see the inheritance column ("I"). @@ -285,7 +278,7 @@ sub-slots +---------------------------------------------------------+-----------------------------------+--------------+ | :c:member:`~PyNumberMethods.nb_inplace_true_divide` | :c:type:`binaryfunc` | __truediv__ | +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_index` | :c:type:`binaryfunc` | __index__ | + | :c:member:`~PyNumberMethods.nb_index` | :c:type:`unaryfunc` | __index__ | +---------------------------------------------------------+-----------------------------------+--------------+ | :c:member:`~PyNumberMethods.nb_matrix_multiply` | :c:type:`binaryfunc` | __matmul__ | | | | __rmatmul__ | @@ -364,12 +357,6 @@ slot typedefs +-----------------------------+-----------------------------+----------------------+ | :c:type:`reprfunc` | :c:type:`PyObject` * | :c:type:`PyObject` * | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`printfunc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | FILE * | | -| | int | | -+-----------------------------+-----------------------------+----------------------+ | :c:type:`getattrfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | | | :c:type:`PyObject` * | | @@ -497,8 +484,8 @@ type objects) *must* have the :attr:`ob_size` field. PyObject* PyObject._ob_prev These fields are only present when the macro ``Py_TRACE_REFS`` is defined. - Their initialization to *NULL* is taken care of by the ``PyObject_HEAD_INIT`` - macro. For statically allocated objects, these fields always remain *NULL*. + Their initialization to ``NULL`` is taken care of by the ``PyObject_HEAD_INIT`` + macro. For statically allocated objects, these fields always remain ``NULL``. For dynamically allocated objects, these two fields are used to link the object into a doubly-linked list of *all* live objects on the heap. This could be used for various debugging purposes; currently the only use is to print the objects @@ -529,7 +516,7 @@ type objects) *must* have the :attr:`ob_size` field. argument to the ``PyObject_HEAD_INIT`` macro, and its value should normally be ``&PyType_Type``. However, for dynamically loadable extension modules that must be usable on Windows (at least), the compiler complains that this is not a valid - initializer. Therefore, the convention is to pass *NULL* to the + initializer. Therefore, the convention is to pass ``NULL`` to the ``PyObject_HEAD_INIT`` macro and to initialize this field explicitly at the start of the module's initialization function, before doing anything else. This is typically done like this:: @@ -537,7 +524,7 @@ type objects) *must* have the :attr:`ob_size` field. Foo_Type.ob_type = &PyType_Type; This should be done before any instances of the type are created. - :c:func:`PyType_Ready` checks if :attr:`ob_type` is *NULL*, and if so, + :c:func:`PyType_Ready` checks if :attr:`ob_type` is ``NULL``, and if so, initializes it to the :attr:`ob_type` field of the base class. :c:func:`PyType_Ready` will not change this field if it is non-zero. @@ -563,7 +550,7 @@ PyTypeObject Slots ------------------ Each slot has a section describing inheritance. If :c:func:`PyType_Ready` -may set a value when the field is set to *NULL* then there will also be +may set a value when the field is set to ``NULL`` then there will also be a "Default" section. (Note that many fields set on :c:type:`PyBaseObject_Type` and :c:type:`PyType_Type` effectively act as defaults.) @@ -592,7 +579,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) type will be impossible to pickle. Additionally, it will not be listed in module documentations created with pydoc. - This field must not be *NULL*. It is the only required field + This field must not be ``NULL``. It is the only required field in :c:func:`PyTypeObject` (other than potentially :c:member:`~PyTypeObject.tp_itemsize`). @@ -637,7 +624,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the alignment requirement for ``double``). - For any type with variable-length instances, this field must not be *NULL*. + For any type with variable-length instances, this field must not be ``NULL``. **Inheritance:** @@ -660,9 +647,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) the instance is still in existence, but there are no references to it. The destructor function should free all references which the instance owns, free all memory buffers owned by the instance (using the freeing function corresponding - to the allocation function used to allocate the buffer), and finally (as its - last action) call the type's :c:member:`~PyTypeObject.tp_free` function. If the type is not - subtypable (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is + to the allocation function used to allocate the buffer), and call the type's + :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable + (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is permissible to call the object deallocator directly instead of via :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the instance; this is normally :c:func:`PyObject_Del` if the instance was allocated @@ -670,14 +657,74 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:func:`PyObject_GC_Del` if the instance was allocated using :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`. + Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the + deallocator should decrement the reference count for its type object after + calling the type deallocator. In order to avoid dangling pointers, the + recommended way to achieve this is: + + .. code-block:: c + + static void foo_dealloc(foo_object *self) { + PyTypeObject *tp = Py_TYPE(self); + // free references and buffers here + tp->tp_free(self); + Py_DECREF(tp); + } + + **Inheritance:** This field is inherited by subtypes. -.. c:member:: printfunc PyTypeObject.tp_print +.. c:member:: Py_ssize_t PyTypeObject.tp_vectorcall_offset + + An optional offset to a per-instance function that implements calling + the object using the :ref:`vectorcall protocol `, + a more efficient alternative + of the simpler :c:member:`~PyTypeObject.tp_call`. + + This field is only used if the flag :const:`Py_TPFLAGS_HAVE_VECTORCALL` + is set. If so, this must be a positive integer containing the offset in the + instance of a :c:type:`vectorcallfunc` pointer. + + The *vectorcallfunc* pointer may be ``NULL``, in which case the instance behaves + as if :const:`Py_TPFLAGS_HAVE_VECTORCALL` was not set: calling the instance + falls back to :c:member:`~PyTypeObject.tp_call`. + + Any class that sets ``Py_TPFLAGS_HAVE_VECTORCALL`` must also set + :c:member:`~PyTypeObject.tp_call` and make sure its behaviour is consistent + with the *vectorcallfunc* function. + This can be done by setting *tp_call* to :c:func:`PyVectorcall_Call`. + + .. warning:: + + It is not recommended for :ref:`heap types ` to implement + the vectorcall protocol. + When a user sets :attr:`__call__` in Python code, only *tp_call* is updated, + likely making it inconsistent with the vectorcall function. + + .. note:: + + The semantics of the ``tp_vectorcall_offset`` slot are provisional and + expected to be finalized in Python 3.9. + If you use vectorcall, plan for updating your code for Python 3.9. + + .. versionchanged:: 3.8 + + Before version 3.8, this slot was named ``tp_print``. + In Python 2.x, it was used for printing to a file. + In Python 3.0 to 3.7, it was unused. - Reserved slot, formerly used for print formatting in Python 2.x. + **Inheritance:** + + This field is always inherited. + However, the :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag is not + always inherited. If it's not, then the subclass won't use + :ref:`vectorcall `, except when + :c:func:`PyVectorcall_Call` is explicitly called. + This is in particular the case for `heap types`_ + (including subclasses defined in Python). .. c:member:: getattrfunc PyTypeObject.tp_getattr @@ -694,7 +741,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattro`: a subtype inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. + the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both ``NULL``. .. c:member:: setattrfunc PyTypeObject.tp_setattr @@ -711,7 +758,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattro`: a subtype inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. + the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both ``NULL``. .. c:member:: PyAsyncMethods* PyTypeObject.tp_as_async @@ -828,13 +875,13 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_richcompare`: a subtype inherits both of :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash`, when the subtype's - :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both *NULL*. + :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both ``NULL``. .. c:member:: ternaryfunc PyTypeObject.tp_call An optional pointer to a function that implements calling the object. This - should be *NULL* if the object is not callable. The signature is the same as + should be ``NULL`` if the object is not callable. The signature is the same as for :c:func:`PyObject_Call`:: PyObject *tp_call(PyObject *self, PyObject *args, PyObject *kwargs); @@ -886,7 +933,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattr`: a subtype inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. + the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both ``NULL``. **Default:** @@ -899,9 +946,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) The signature is the same as for :c:func:`PyObject_SetAttr`:: - PyObject *tp_setattro(PyObject *self, PyObject *attr, PyObject *value); + int tp_setattro(PyObject *self, PyObject *attr, PyObject *value); - In addition, setting *value* to *NULL* to delete an attribute must be + In addition, setting *value* to ``NULL`` to delete an attribute must be supported. It is usually convenient to set this field to :c:func:`PyObject_GenericSetAttr`, which implements the normal way of setting object attributes. @@ -912,7 +959,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattr`: a subtype inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. + the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both ``NULL``. **Default:** @@ -939,7 +986,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_as_number`, :c:member:`~PyTypeObject.tp_as_sequence`, :c:member:`~PyTypeObject.tp_as_mapping`, and :c:member:`~PyTypeObject.tp_as_buffer`) that were historically not always present are valid; if such a flag bit is clear, the type fields it guards must not be accessed and - must be considered to have a zero or *NULL* value instead. + must be considered to have a zero or ``NULL`` value instead. **Inheritance:** @@ -952,7 +999,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields, i.e. if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have - *NULL* values. + ``NULL`` values. .. XXX are most flag bits *really* inherited individually? @@ -970,7 +1017,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. data:: Py_TPFLAGS_HEAPTYPE - This bit is set when the type object itself is allocated on the heap. In this + This bit is set when the type object itself is allocated on the heap, for + example, types created dynamically using :c:func:`PyType_FromSpec`. In this case, the :attr:`ob_type` field of its instances is considered a reference to the type, and the type object is INCREF'ed when a new instance is created, and DECREF'ed when an instance is destroyed (this does not apply to instances of @@ -1030,7 +1078,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) together with the :attr:`tp_traverse` and :attr:`tp_clear` fields, i.e. if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the :attr:`tp_traverse` and - :attr:`tp_clear` fields in the subtype exist and have *NULL* + :attr:`tp_clear` fields in the subtype exist and have ``NULL`` values. @@ -1045,6 +1093,32 @@ and :c:type:`PyType_Type` effectively act as defaults.) ??? + + .. data:: Py_TPFLAGS_METHOD_DESCRIPTOR + + This bit indicates that objects behave like unbound methods. + + If this flag is set for ``type(meth)``, then: + + - ``meth.__get__(obj, cls)(*args, **kwds)`` (with ``obj`` not None) + must be equivalent to ``meth(obj, *args, **kwds)``. + + - ``meth.__get__(None, cls)(*args, **kwds)`` + must be equivalent to ``meth(*args, **kwds)``. + + This flag enables an optimization for typical method calls like + ``obj.meth()``: it avoids creating a temporary "bound method" object for + ``obj.meth``. + + .. versionadded:: 3.8 + + **Inheritance:** + + This flag is never inherited by heap types. + For extension types, it is inherited whenever + :c:member:`~PyTypeObject.tp_descr_get` is inherited. + + .. XXX Document more flags here? @@ -1073,6 +1147,26 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.4 + .. deprecated:: 3.8 + This flag isn't necessary anymore, as the interpreter assumes the + :c:member:`~PyTypeObject.tp_finalize` slot is always present in the + type structure. + + + .. data:: Py_TPFLAGS_HAVE_VECTORCALL + + This bit is set when the class implements + the :ref:`vectorcall protocol `. + See :c:member:`~PyTypeObject.tp_vectorcall_offset` for details. + + **Inheritance:** + + This bit is inherited for *static* subtypes if + :c:member:`~PyTypeObject.tp_call` is also inherited. + `Heap types`_ do not inherit ``Py_TPFLAGS_HAVE_VECTORCALL``. + + .. versionadded:: 3.9 + .. c:member:: const char* PyTypeObject.tp_doc @@ -1098,7 +1192,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python - objects. For example, this is function :c:func:`local_traverse` from the + objects that the instance owns. For example, this is function :c:func:`local_traverse` from the :mod:`_thread` extension module:: static int @@ -1112,16 +1206,42 @@ and :c:type:`PyType_Type` effectively act as defaults.) Note that :c:func:`Py_VISIT` is called only on those members that can participate in reference cycles. Although there is also a ``self->key`` member, it can only - be *NULL* or a Python string and therefore cannot be part of a reference cycle. + be ``NULL`` or a Python string and therefore cannot be part of a reference cycle. On the other hand, even if you know a member can never be part of a cycle, as a debugging aid you may want to visit it anyway just so the :mod:`gc` module's :func:`~gc.get_referents` function will include it. + .. warning:: + When implementing :c:member:`~PyTypeObject.tp_traverse`, only the members + that the instance *owns* (by having strong references to them) must be + visited. For instance, if an object supports weak references via the + :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting + the linked list (what *tp_weaklist* points to) must **not** be + visited as the instance does not directly own the weak references to itself + (the weakreference list is there to support the weak reference machinery, + but the instance has no strong reference to the elements inside it, as they + are allowed to be removed even if the instance is still alive). + Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to :c:func:`local_traverse` to have these specific names; don't name them just anything. + Heap-allocated types (:const:`Py_TPFLAGS_HEAPTYPE`, such as those created + with :c:func:`PyType_FromSpec` and similar APIs) hold a reference to their + type. Their traversal function must therefore either visit + :c:func:`Py_TYPE(self) `, or delegate this responsibility by + calling ``tp_traverse`` of another heap-allocated type (such as a + heap-allocated superclass). + If they do not, the type object may not be garbage-collected. + + .. versionchanged:: 3.9 + + Heap-allocated types are expected to visit ``Py_TYPE(self)`` in + ``tp_traverse``. In earlier versions of Python, due to + `bug 40217 `_, doing this + may lead to crashes in subclasses. + **Inheritance:** Group: :const:`Py_TPFLAGS_HAVE_GC`, :attr:`tp_traverse`, :attr:`tp_clear` @@ -1151,7 +1271,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to those of its members that may be Python objects, and set its pointers to those - members to *NULL*, as in the following example:: + members to ``NULL``, as in the following example:: static int local_clear(localobject *self) @@ -1165,12 +1285,12 @@ and :c:type:`PyType_Type` effectively act as defaults.) The :c:func:`Py_CLEAR` macro should be used, because clearing references is delicate: the reference to the contained object must not be decremented until - after the pointer to the contained object is set to *NULL*. This is because + after the pointer to the contained object is set to ``NULL``. This is because decrementing the reference count may cause the contained object to become trash, triggering a chain of reclamation activity that may include invoking arbitrary Python code (due to finalizers, or weakref callbacks, associated with the contained object). If it's possible for such code to reference *self* again, - it's important that the pointer to the contained object be *NULL* at that time, + it's important that the pointer to the contained object be ``NULL`` at that time, so that *self* knows the contained object can no longer be used. The :c:func:`Py_CLEAR` macro performs the operations in a safe order. @@ -1204,15 +1324,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) The function should return the result of the comparison (usually ``Py_True`` or ``Py_False``). If the comparison is undefined, it must return - ``Py_NotImplemented``, if another error occurred it must return *NULL* and + ``Py_NotImplemented``, if another error occurred it must return ``NULL`` and set an exception condition. - .. note:: - - If you want to implement a type for which only a limited set of - comparisons makes sense (e.g. ``==`` and ``!=``, but not ``<`` and - friends), directly raise :exc:`TypeError` in the rich comparison function. - The following constants are defined to be used as the third argument for :c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`: @@ -1234,7 +1348,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The following macro is defined to ease writing rich comparison functions: - .. c:function:: PyObject \*Py_RETURN_RICHCOMPARE(VAL_A, VAL_B, int op) + .. c:macro:: Py_RETURN_RICHCOMPARE(VAL_A, VAL_B, op) Return ``Py_True`` or ``Py_False`` from the function, depending on the result of a comparison. @@ -1244,7 +1358,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The return value's reference count is properly incremented. - On error, sets an exception and returns *NULL* from the function. + On error, sets an exception and returns ``NULL`` from the function. .. versionadded:: 3.7 @@ -1255,7 +1369,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_hash`: a subtype inherits :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` when the subtype's :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both - *NULL*. + ``NULL``. **Default:** @@ -1272,8 +1386,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) than zero and contains the offset in the instance structure of the weak reference list head (ignoring the GC header, if present); this offset is used by :c:func:`PyObject_ClearWeakRefs` and the :c:func:`PyWeakref_\*` functions. The - instance structure needs to include a field of type :c:type:`PyObject\*` which is - initialized to *NULL*. + instance structure needs to include a field of type :c:type:`PyObject*` which is + initialized to ``NULL``. Do not confuse this field with :c:member:`~PyTypeObject.tp_weaklist`; that is the list head for weak references to the type object itself. @@ -1322,9 +1436,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) PyObject *tp_iternext(PyObject *self); - When the iterator is exhausted, it must return *NULL*; a :exc:`StopIteration` + When the iterator is exhausted, it must return ``NULL``; a :exc:`StopIteration` exception may or may not be set. When another error occurs, it must return - *NULL* too. Its presence signals that the instances of this type are + ``NULL`` too. Its presence signals that the instances of this type are iterators. Iterator types should also define the :c:member:`~PyTypeObject.tp_iter` function, and that @@ -1340,7 +1454,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMethodDef* PyTypeObject.tp_methods - An optional pointer to a static *NULL*-terminated array of :c:type:`PyMethodDef` + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMethodDef` structures, declaring regular methods of this type. For each entry in the array, an entry is added to the type's dictionary (see @@ -1354,7 +1468,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMemberDef* PyTypeObject.tp_members - An optional pointer to a static *NULL*-terminated array of :c:type:`PyMemberDef` + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMemberDef` structures, declaring regular data members (fields or slots) of instances of this type. @@ -1369,7 +1483,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset - An optional pointer to a static *NULL*-terminated array of :c:type:`PyGetSetDef` + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyGetSetDef` structures, declaring computed attributes of instances of this type. For each entry in the array, an entry is added to the type's dictionary (see @@ -1419,7 +1533,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The type's dictionary is stored here by :c:func:`PyType_Ready`. - This field should normally be initialized to *NULL* before PyType_Ready is + This field should normally be initialized to ``NULL`` before PyType_Ready is called; it may also be initialized to a dictionary containing initial attributes for the type. Once :c:func:`PyType_Ready` has initialized the type, extra attributes for the type may be added to this dictionary only if they don't @@ -1432,7 +1546,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** - If this field is *NULL*, :c:func:`PyType_Ready` will assign a new + If this field is ``NULL``, :c:func:`PyType_Ready` will assign a new dictionary to it. .. warning:: @@ -1465,7 +1579,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) int tp_descr_set(PyObject *self, PyObject *obj, PyObject *value); - The *value* argument is set to *NULL* to delete the value. + The *value* argument is set to ``NULL`` to delete the value. .. XXX explain more? @@ -1531,7 +1645,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** This slot has no default. For static types, if the field is - *NULL* then no :attr:`__dict__` gets created for instances. + ``NULL`` then no :attr:`__dict__` gets created for instances. .. c:member:: initproc PyTypeObject.tp_init @@ -1551,7 +1665,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) arguments represent positional and keyword arguments of the call to :meth:`__init__`. - The :c:member:`~PyTypeObject.tp_init` function, if not *NULL*, is called when an instance is + The :c:member:`~PyTypeObject.tp_init` function, if not ``NULL``, is called when an instance is created normally by calling its type, after the type's :c:member:`~PyTypeObject.tp_new` function has returned an instance of the type. If the :c:member:`~PyTypeObject.tp_new` function returns an instance of some other type that is not a subtype of the original type, no @@ -1601,9 +1715,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) PyObject *tp_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds); - The subtype argument is the type of the object being created; the *args* and + The *subtype* argument is the type of the object being created; the *args* and *kwds* arguments represent positional and keyword arguments of the call to the - type. Note that subtype doesn't have to equal the type whose :c:member:`~PyTypeObject.tp_new` + type. Note that *subtype* doesn't have to equal the type whose :c:member:`~PyTypeObject.tp_new` function is called; it may be a subtype of that type (but not an unrelated type). @@ -1618,12 +1732,12 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** This field is inherited by subtypes, except it is not inherited by static types - whose :c:member:`~PyTypeObject.tp_base` is *NULL* or ``&PyBaseObject_Type``. + whose :c:member:`~PyTypeObject.tp_base` is ``NULL`` or ``&PyBaseObject_Type``. **Default:** For static types this field has no default. This means if the - slot is defined as *NULL*, the type cannot be called to create new + slot is defined as ``NULL``, the type cannot be called to create new instances; presumably there is some other way to create instances, like a factory function. @@ -1674,7 +1788,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** - This slot has no default. If this field is *NULL*, + This slot has no default. If this field is ``NULL``, :const:`Py_TPFLAGS_HAVE_GC` is used as the functional equivalent. @@ -1682,7 +1796,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) Tuple of base types. - This is set for types created by a class statement. It should be *NULL* for + This is set for types created by a class statement. It should be ``NULL`` for statically defined types. **Inheritance:** @@ -1786,30 +1900,20 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. seealso:: "Safe object finalization" (:pep:`442`) -The remaining fields are only defined if the feature test macro -:const:`COUNT_ALLOCS` is defined, and are for internal use only. They are -documented here for completeness. None of these fields are inherited by -subtypes. - -.. c:member:: Py_ssize_t PyTypeObject.tp_allocs - - Number of allocations. +.. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall -.. c:member:: Py_ssize_t PyTypeObject.tp_frees + Vectorcall function to use for calls of this type object. + In other words, it is used to implement + :ref:`vectorcall ` for ``type.__call__``. + If ``tp_vectorcall`` is ``NULL``, the default call implementation + using :attr:`__new__` and :attr:`__init__` is used. - Number of frees. - -.. c:member:: Py_ssize_t PyTypeObject.tp_maxalloc - - Maximum simultaneously allocated objects. - -.. c:member:: PyTypeObject* PyTypeObject.tp_prev + **Inheritance:** - Pointer to the previous type object with a non-zero :c:member:`~PyTypeObject.tp_allocs` field. + This field is never inherited. -.. c:member:: PyTypeObject* PyTypeObject.tp_next + .. versionadded:: 3.9 (the field exists since 3.8 but it's only used since 3.9) - Pointer to the next type object with a non-zero :c:member:`~PyTypeObject.tp_allocs` field. Also, note that, in a garbage collected Python, :c:member:`~PyTypeObject.tp_dealloc` may be called from any Python thread, not just the thread which created the object (if the object @@ -1822,16 +1926,35 @@ objects on the thread which called tp_dealloc will not violate any assumptions of the library. +.. _heap-types: + Heap Types ---------- -In addition to defining Python types statically, you can define them -dynamically (i.e. to the heap) using :c:func:`PyType_FromSpec` and -:c:func:`PyType_FromSpecWithBases`. +Traditionally, types defined in C code are *static*, that is, +a static :c:type:`PyTypeObject` structure is defined directly in code +and initialized using :c:func:`PyType_Ready`. + +This results in types that are limited relative to types defined in Python: + +* Static types are limited to one base, i.e. they cannot use multiple + inheritance. +* Static type objects (but not necessarily their instances) are immutable. + It is not possible to add or modify the type object's attributes from Python. +* Static type objects are shared across + :ref:`sub-interpreters `, so they should not + include any subinterpreter-specific state. -.. XXX Explain how to use PyType_FromSpec(). +Also, since :c:type:`PyTypeObject` is not part of the :ref:`stable ABI `, +any extension modules using static types must be compiled for a specific +Python minor version. -.. XXX Document PyType_Spec. +An alternative to static types is *heap-allocated types*, or *heap types* +for short, which correspond closely to classes created by Python's +``class`` statement. + +This is done by filling a :c:type:`PyType_Spec` structure and calling +:c:func:`PyType_FromSpecWithBases`. .. _number-structs: @@ -1901,12 +2024,12 @@ Number Object Structures and implement the necessary conversions (at least one of the operands is an instance of the defined type). If the operation is not defined for the given operands, binary and ternary functions must return - ``Py_NotImplemented``, if another error occurred they must return *NULL* + ``Py_NotImplemented``, if another error occurred they must return ``NULL`` and set an exception. .. note:: - The :c:data:`nb_reserved` field should always be *NULL*. It + The :c:data:`nb_reserved` field should always be ``NULL``. It was previously called :c:data:`nb_long`, and was renamed in Python 3.0.1. @@ -1965,14 +2088,14 @@ Mapping Object Structures This function is used by :c:func:`PyMapping_Size` and :c:func:`PyObject_Size`, and has the same signature. This slot may be set to - *NULL* if the object has no defined length. + ``NULL`` if the object has no defined length. .. c:member:: binaryfunc PyMappingMethods.mp_subscript This function is used by :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetSlice`, and has the same signature as :c:func:`!PyObject_GetItem`. This slot must be filled for the - :c:func:`PyMapping_Check` function to return ``1``, it can be *NULL* + :c:func:`PyMapping_Check` function to return ``1``, it can be ``NULL`` otherwise. .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript @@ -1980,8 +2103,8 @@ Mapping Object Structures This function is used by :c:func:`PyObject_SetItem`, :c:func:`PyObject_DelItem`, :c:func:`PyObject_SetSlice` and :c:func:`PyObject_DelSlice`. It has the same signature as - :c:func:`!PyObject_SetItem`, but *v* can also be set to *NULL* to delete - an item. If this slot is *NULL*, the object does not support item + :c:func:`!PyObject_SetItem`, but *v* can also be set to ``NULL`` to delete + an item. If this slot is ``NULL``, the object does not support item assignment and deletion. @@ -2023,11 +2146,11 @@ Sequence Object Structures signature. It is also used by :c:func:`PyObject_GetItem`, after trying the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. This slot must be filled for the :c:func:`PySequence_Check` - function to return ``1``, it can be *NULL* otherwise. + function to return ``1``, it can be ``NULL`` otherwise. Negative indexes are handled as follows: if the :attr:`sq_length` slot is filled, it is called and the sequence length is used to compute a positive - index which is passed to :attr:`sq_item`. If :attr:`sq_length` is *NULL*, + index which is passed to :attr:`sq_item`. If :attr:`sq_length` is ``NULL``, the index is passed as is to the function. .. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item @@ -2036,13 +2159,13 @@ Sequence Object Structures signature. It is also used by :c:func:`PyObject_SetItem` and :c:func:`PyObject_DelItem`, after trying the item assignment and deletion via the :c:member:`~PyMappingMethods.mp_ass_subscript` slot. - This slot may be left to *NULL* if the object does not support + This slot may be left to ``NULL`` if the object does not support item assignment and deletion. .. c:member:: objobjproc PySequenceMethods.sq_contains This function may be used by :c:func:`PySequence_Contains` and has the same - signature. This slot may be left to *NULL*, in this case + signature. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_Contains` simply traverses the sequence until it finds a match. @@ -2050,7 +2173,7 @@ Sequence Object Structures This function is used by :c:func:`PySequence_InPlaceConcat` and has the same signature. It should modify its first operand, and return it. This slot - may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceConcat` + may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceConcat` will fall back to :c:func:`PySequence_Concat`. It is also used by the augmented assignment ``+=``, after trying numeric in-place addition via the :c:member:`~PyNumberMethods.nb_inplace_add` slot. @@ -2059,7 +2182,7 @@ Sequence Object Structures This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same signature. It should modify its first operand, and return it. This slot - may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceRepeat` + may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceRepeat` will fall back to :c:func:`PySequence_Repeat`. It is also used by the augmented assignment ``*=``, after trying numeric in-place multiplication via the :c:member:`~PyNumberMethods.nb_inplace_multiply` slot. @@ -2091,7 +2214,7 @@ Buffer Object Structures steps: (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`, - set :c:data:`view->obj` to *NULL* and return ``-1``. + set :c:data:`view->obj` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. @@ -2137,7 +2260,7 @@ Buffer Object Structures Handle a request to release the resources of the buffer. If no resources need to be released, :c:member:`PyBufferProcs.bf_releasebuffer` may be - *NULL*. Otherwise, a standard implementation of this function will take + ``NULL``. Otherwise, a standard implementation of this function will take these optional steps: (1) Decrement an internal counter for the number of exports. @@ -2191,7 +2314,7 @@ Async Object Structures The returned object must be an iterator, i.e. :c:func:`PyIter_Check` must return ``1`` for it. - This slot may be set to *NULL* if an object is not an :term:`awaitable`. + This slot may be set to ``NULL`` if an object is not an :term:`awaitable`. .. c:member:: unaryfunc PyAsyncMethods.am_aiter @@ -2201,7 +2324,7 @@ Async Object Structures Must return an :term:`awaitable` object. See :meth:`__anext__` for details. - This slot may be set to *NULL* if an object does not implement + This slot may be set to ``NULL`` if an object does not implement asynchronous iteration protocol. .. c:member:: unaryfunc PyAsyncMethods.am_anext @@ -2211,7 +2334,7 @@ Async Object Structures PyObject *am_anext(PyObject *self); Must return an :term:`awaitable` object. See :meth:`__anext__` for details. - This slot may be set to *NULL*. + This slot may be set to ``NULL``. .. _slot-typedefs: @@ -2252,10 +2375,6 @@ Slot Type typedefs See :c:member:`~PyTypeObject.tp_repr`. -.. c:type:: int (*printfunc)(PyObject *, FILE *, int) - - This is hidden if :const:`PY_LIMITED_API` is set. - .. c:type:: PyObject *(*getattrfunc)(PyObject *self, char *attr) Return the value of the named attribute for the object. @@ -2263,7 +2382,7 @@ Slot Type typedefs .. c:type:: int (*setattrfunc)(PyObject *self, char *attr, PyObject *value) Set the value of the named attribute for the object. - The value argument is set to *NULL* to delete the attribute. + The value argument is set to ``NULL`` to delete the attribute. .. c:type:: PyObject *(*getattrofunc)(PyObject *self, PyObject *attr) @@ -2274,7 +2393,7 @@ Slot Type typedefs .. c:type:: int (*setattrofunc)(PyObject *self, PyObject *attr, PyObject *value) Set the value of the named attribute for the object. - The value argument is set to *NULL* to delete the attribute. + The value argument is set to ``NULL`` to delete the attribute. See :c:member:`~PyTypeObject.tp_setattro`. @@ -2323,7 +2442,7 @@ Slot Type typedefs .. c:type:: int (*objobjargproc)(PyObject *, PyObject *, PyObject *) -.. _examples: +.. _typedef-examples: Examples ======== @@ -2359,7 +2478,7 @@ with a more verbose initializer:: sizeof(MyObject), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)myobj_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 97c5ebcb10953c..5518214a793e0c 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _unicodeobjects: @@ -22,18 +22,23 @@ in the Unicode object. The :c:type:`Py_UNICODE*` representation is deprecated and inefficient; it should be avoided in performance- or memory-sensitive situations. -Due to the transition between the old APIs and the new APIs, unicode objects +Due to the transition between the old APIs and the new APIs, Unicode objects can internally be in two states depending on how they were created: -* "canonical" unicode objects are all objects created by a non-deprecated - unicode API. They use the most efficient representation allowed by the +* "canonical" Unicode objects are all objects created by a non-deprecated + Unicode API. They use the most efficient representation allowed by the implementation. -* "legacy" unicode objects have been created through one of the deprecated +* "legacy" Unicode objects have been created through one of the deprecated APIs (typically :c:func:`PyUnicode_FromUnicode`) and only bear the :c:type:`Py_UNICODE*` representation; you will have to call :c:func:`PyUnicode_READY` on them before calling any other API. +.. note:: + The "legacy" Unicode object will be removed in Python 3.12 with deprecated + APIs. All Unicode objects will be "canonical" since then. See :pep:`623` + for more information. + Unicode Type """""""""""" @@ -107,6 +112,9 @@ access internal read-only data of Unicode objects: .. versionadded:: 3.3 + .. deprecated-removed:: 3.10 3.12 + This API will be removed with :c:func:`PyUnicode_FromUnicode`. + .. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o) @@ -138,6 +146,9 @@ access internal read-only data of Unicode objects: .. versionadded:: 3.3 + .. deprecated-removed:: 3.10 3.12 + ``PyUnicode_WCHAR_KIND`` is deprecated. + .. c:function:: int PyUnicode_KIND(PyObject *o) @@ -152,7 +163,7 @@ access internal read-only data of Unicode objects: .. c:function:: void* PyUnicode_DATA(PyObject *o) - Return a void pointer to the raw unicode buffer. *o* has to be a Unicode + Return a void pointer to the raw Unicode buffer. *o* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 @@ -188,7 +199,7 @@ access internal read-only data of Unicode objects: .. versionadded:: 3.3 -.. c:function:: PyUnicode_MAX_CHAR_VALUE(PyObject *o) +.. c:macro:: PyUnicode_MAX_CHAR_VALUE(o) Return the maximum code point that is suitable for creating another string based on *o*, which must be in the "canonical" representation. This is @@ -197,18 +208,13 @@ access internal read-only data of Unicode objects: .. versionadded:: 3.3 -.. c:function:: int PyUnicode_ClearFreeList() - - Clear the free list. Return the total number of freed items. - - .. c:function:: Py_ssize_t PyUnicode_GET_SIZE(PyObject *o) Return the size of the deprecated :c:type:`Py_UNICODE` representation, in code units (this includes surrogate pairs as 2 units). *o* has to be a Unicode object (not checked). - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style Unicode API, please migrate to using :c:func:`PyUnicode_GET_LENGTH`. @@ -218,7 +224,7 @@ access internal read-only data of Unicode objects: Return the size of the deprecated :c:type:`Py_UNICODE` representation in bytes. *o* has to be a Unicode object (not checked). - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style Unicode API, please migrate to using :c:func:`PyUnicode_GET_LENGTH`. @@ -236,15 +242,25 @@ access internal read-only data of Unicode objects: .. versionchanged:: 3.3 This macro is now inefficient -- because in many cases the :c:type:`Py_UNICODE` representation does not exist and needs to be created - -- and can fail (return *NULL* with an exception set). Try to port the + -- and can fail (return ``NULL`` with an exception set). Try to port the code to use the new :c:func:`PyUnicode_nBYTE_DATA` macros or use :c:func:`PyUnicode_WRITE` or :c:func:`PyUnicode_READ`. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style Unicode API, please migrate to using the :c:func:`PyUnicode_nBYTE_DATA` family of macros. +.. c:function:: int PyUnicode_IsIdentifier(PyObject *o) + + Return ``1`` if the string is a valid identifier according to the language + definition, section :ref:`identifiers`. Return ``0`` otherwise. + + .. versionchanged:: 3.9 + The function does not call :c:func:`Py_FatalError` anymore if the string + is not ready. + + Unicode Character Properties """""""""""""""""""""""""""" @@ -413,11 +429,11 @@ APIs: Create a Unicode object from the char buffer *u*. The bytes will be interpreted as being UTF-8 encoded. The buffer is copied into the new - object. If the buffer is not *NULL*, the return value might be a shared + object. If the buffer is not ``NULL``, the return value might be a shared object, i.e. modification of the data is not allowed. - If *u* is *NULL*, this function behaves like :c:func:`PyUnicode_FromUnicode` - with the buffer set to *NULL*. This usage is deprecated in favor of + If *u* is ``NULL``, this function behaves like :c:func:`PyUnicode_FromUnicode` + with the buffer set to ``NULL``. This usage is deprecated in favor of :c:func:`PyUnicode_New`. @@ -430,7 +446,7 @@ APIs: .. c:function:: PyObject* PyUnicode_FromFormat(const char *format, ...) Take a C :c:func:`printf`\ -style *format* string and a variable number of - arguments, calculate the size of the resulting Python unicode string and return + arguments, calculate the size of the resulting Python Unicode string and return a string with the values formatted into it. The variable arguments must be C types and must correspond exactly to the format characters in the *format* ASCII-encoded string. The following format characters are allowed: @@ -443,82 +459,82 @@ APIs: .. tabularcolumns:: |l|l|L| - +-------------------+---------------------+--------------------------------+ - | Format Characters | Type | Comment | - +===================+=====================+================================+ - | :attr:`%%` | *n/a* | The literal % character. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%c` | int | A single character, | - | | | represented as a C int. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%d` | int | Equivalent to | - | | | ``printf("%d")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%u` | unsigned int | Equivalent to | - | | | ``printf("%u")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%ld` | long | Equivalent to | - | | | ``printf("%ld")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%li` | long | Equivalent to | - | | | ``printf("%li")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%lu` | unsigned long | Equivalent to | - | | | ``printf("%lu")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%lld` | long long | Equivalent to | - | | | ``printf("%lld")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%lli` | long long | Equivalent to | - | | | ``printf("%lli")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%llu` | unsigned long long | Equivalent to | - | | | ``printf("%llu")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%zd` | Py_ssize_t | Equivalent to | - | | | ``printf("%zd")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%zi` | Py_ssize_t | Equivalent to | - | | | ``printf("%zi")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%zu` | size_t | Equivalent to | - | | | ``printf("%zu")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%i` | int | Equivalent to | - | | | ``printf("%i")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%x` | int | Equivalent to | - | | | ``printf("%x")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%s` | const char\* | A null-terminated C character | - | | | array. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%p` | const void\* | The hex representation of a C | - | | | pointer. Mostly equivalent to | - | | | ``printf("%p")`` except that | - | | | it is guaranteed to start with | - | | | the literal ``0x`` regardless | - | | | of what the platform's | - | | | ``printf`` yields. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%A` | PyObject\* | The result of calling | - | | | :func:`ascii`. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%U` | PyObject\* | A unicode object. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%V` | PyObject\*, | A unicode object (which may be | - | | const char\* | *NULL*) and a null-terminated | - | | | C character array as a second | - | | | parameter (which will be used, | - | | | if the first parameter is | - | | | *NULL*). | - +-------------------+---------------------+--------------------------------+ - | :attr:`%S` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Str`. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%R` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Repr`. | - +-------------------+---------------------+--------------------------------+ + +-------------------+---------------------+----------------------------------+ + | Format Characters | Type | Comment | + +===================+=====================+==================================+ + | :attr:`%%` | *n/a* | The literal % character. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%c` | int | A single character, | + | | | represented as a C int. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%d` | int | Equivalent to | + | | | ``printf("%d")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%u` | unsigned int | Equivalent to | + | | | ``printf("%u")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%ld` | long | Equivalent to | + | | | ``printf("%ld")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%li` | long | Equivalent to | + | | | ``printf("%li")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%lu` | unsigned long | Equivalent to | + | | | ``printf("%lu")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%lld` | long long | Equivalent to | + | | | ``printf("%lld")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%lli` | long long | Equivalent to | + | | | ``printf("%lli")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%llu` | unsigned long long | Equivalent to | + | | | ``printf("%llu")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%zd` | Py_ssize_t | Equivalent to | + | | | ``printf("%zd")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%zi` | Py_ssize_t | Equivalent to | + | | | ``printf("%zi")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%zu` | size_t | Equivalent to | + | | | ``printf("%zu")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%i` | int | Equivalent to | + | | | ``printf("%i")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%x` | int | Equivalent to | + | | | ``printf("%x")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%s` | const char\* | A null-terminated C character | + | | | array. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%p` | const void\* | The hex representation of a C | + | | | pointer. Mostly equivalent to | + | | | ``printf("%p")`` except that | + | | | it is guaranteed to start with | + | | | the literal ``0x`` regardless | + | | | of what the platform's | + | | | ``printf`` yields. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%A` | PyObject\* | The result of calling | + | | | :func:`ascii`. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%U` | PyObject\* | A Unicode object. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%V` | PyObject\*, | A Unicode object (which may be | + | | const char\* | ``NULL``) and a null-terminated | + | | | C character array as a second | + | | | parameter (which will be used, | + | | | if the first parameter is | + | | | ``NULL``). | + +-------------------+---------------------+----------------------------------+ + | :attr:`%S` | PyObject\* | The result of calling | + | | | :c:func:`PyObject_Str`. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%R` | PyObject\* | The result of calling | + | | | :c:func:`PyObject_Repr`. | + +-------------------+---------------------+----------------------------------+ An unrecognized format character causes all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. @@ -526,9 +542,9 @@ APIs: .. note:: The width formatter unit is number of characters rather than bytes. The precision formatter unit is number of bytes for ``"%s"`` and - ``"%V"`` (if the ``PyObject*`` argument is NULL), and a number of + ``"%V"`` (if the ``PyObject*`` argument is ``NULL``), and a number of characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"`` - (if the ``PyObject*`` argument is not NULL). + (if the ``PyObject*`` argument is not ``NULL``). .. [1] For integer specifiers (d, u, ld, li, lu, lld, lli, llu, zd, zi, zu, i, x): the 0-conversion flag has effect even when a precision is given. @@ -558,13 +574,13 @@ APIs: :class:`bytes`, :class:`bytearray` and other :term:`bytes-like objects ` are decoded according to the given *encoding* and using the error handling - defined by *errors*. Both can be *NULL* to have the interface use the default + defined by *errors*. Both can be ``NULL`` to have the interface use the default values (see :ref:`builtincodecs` for details). All other objects, including Unicode objects, cause a :exc:`TypeError` to be set. - The API returns *NULL* if there was an error. The caller is responsible for + The API returns ``NULL`` if there was an error. The caller is responsible for decref'ing the returned objects. @@ -640,7 +656,7 @@ APIs: Py_ssize_t buflen, int copy_null) Copy the string *u* into a UCS4 buffer, including a null character, if - *copy_null* is set. Returns *NULL* and sets an exception on error (in + *copy_null* is set. Returns ``NULL`` and sets an exception on error (in particular, a :exc:`SystemError` if *buflen* is smaller than the length of *u*). *buffer* is returned on success. @@ -650,7 +666,7 @@ APIs: .. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *u) Copy the string *u* into a new UCS4 buffer that is allocated using - :c:func:`PyMem_Malloc`. If this fails, *NULL* is returned with a + :c:func:`PyMem_Malloc`. If this fails, ``NULL`` is returned with a :exc:`MemoryError` set. The returned buffer always has an extra null code point appended. @@ -670,42 +686,45 @@ Extension modules can continue using them, as they will not be removed in Python .. c:function:: PyObject* PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size) Create a Unicode object from the Py_UNICODE buffer *u* of the given size. *u* - may be *NULL* which causes the contents to be undefined. It is the user's + may be ``NULL`` which causes the contents to be undefined. It is the user's responsibility to fill in the needed data. The buffer is copied into the new object. - If the buffer is not *NULL*, the return value might be a shared object. + If the buffer is not ``NULL``, the return value might be a shared object. Therefore, modification of the resulting Unicode object is only allowed when - *u* is *NULL*. + *u* is ``NULL``. - If the buffer is *NULL*, :c:func:`PyUnicode_READY` must be called once the + If the buffer is ``NULL``, :c:func:`PyUnicode_READY` must be called once the string content has been filled before using any of the access macros such as :c:func:`PyUnicode_KIND`. - Please migrate to using :c:func:`PyUnicode_FromKindAndData`, - :c:func:`PyUnicode_FromWideChar` or :c:func:`PyUnicode_New`. + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_FromKindAndData`, :c:func:`PyUnicode_FromWideChar`, or + :c:func:`PyUnicode_New`. .. c:function:: Py_UNICODE* PyUnicode_AsUnicode(PyObject *unicode) Return a read-only pointer to the Unicode object's internal - :c:type:`Py_UNICODE` buffer, or *NULL* on error. This will create the + :c:type:`Py_UNICODE` buffer, or ``NULL`` on error. This will create the :c:type:`Py_UNICODE*` representation of the object if it is not yet available. The buffer is always terminated with an extra null code point. Note that the resulting :c:type:`Py_UNICODE` string may also contain embedded null code points, which would cause the string to be truncated when used in most C functions. - Please migrate to using :c:func:`PyUnicode_AsUCS4`, - :c:func:`PyUnicode_AsWideChar`, :c:func:`PyUnicode_ReadChar` or similar new - APIs. + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsWideChar`, + :c:func:`PyUnicode_ReadChar` or similar new APIs. .. c:function:: PyObject* PyUnicode_TransformDecimalToASCII(Py_UNICODE *s, Py_ssize_t size) Create a Unicode object by replacing all decimal digits in :c:type:`Py_UNICODE` buffer of the given *size* by ASCII digits 0--9 - according to their decimal value. Return *NULL* if an exception occurs. + according to their decimal value. Return ``NULL`` if an exception occurs. .. c:function:: Py_UNICODE* PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) @@ -718,19 +737,10 @@ Extension modules can continue using them, as they will not be removed in Python .. versionadded:: 3.3 - -.. c:function:: Py_UNICODE* PyUnicode_AsUnicodeCopy(PyObject *unicode) - - Create a copy of a Unicode string ending with a null code point. Return *NULL* - and raise a :exc:`MemoryError` exception on memory allocation failure, - otherwise return a new allocated buffer (use :c:func:`PyMem_Free` to free - the buffer). Note that the resulting :c:type:`Py_UNICODE*` string may - contain embedded null code points, which would cause the string to be - truncated when used in most C functions. - - .. versionadded:: 3.2 - - Please migrate to using :c:func:`PyUnicode_AsUCS4Copy` or similar new APIs. + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsWideChar`, + :c:func:`PyUnicode_ReadChar` or similar new APIs. .. c:function:: Py_ssize_t PyUnicode_GetSize(PyObject *unicode) @@ -738,7 +748,9 @@ Extension modules can continue using them, as they will not be removed in Python Return the size of the deprecated :c:type:`Py_UNICODE` representation, in code units (this includes surrogate pairs as 2 units). - Please migrate to using :c:func:`PyUnicode_GetLength`. + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_GET_LENGTH`. .. c:function:: PyObject* PyUnicode_FromObject(PyObject *obj) @@ -932,7 +944,7 @@ wchar_t Support Create a Unicode object from the :c:type:`wchar_t` buffer *w* of the given *size*. Passing ``-1`` as the *size* indicates that the function must itself compute the length, using wcslen. - Return *NULL* on failure. + Return ``NULL`` on failure. .. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size) @@ -951,22 +963,22 @@ wchar_t Support .. c:function:: wchar_t* PyUnicode_AsWideCharString(PyObject *unicode, Py_ssize_t *size) Convert the Unicode object to a wide character string. The output string - always ends with a null character. If *size* is not *NULL*, write the number + always ends with a null character. If *size* is not ``NULL``, write the number of wide characters (excluding the trailing null termination character) into *\*size*. Note that the resulting :c:type:`wchar_t` string might contain null characters, which would cause the string to be truncated when used with - most C functions. If *size* is *NULL* and the :c:type:`wchar_t*` string + most C functions. If *size* is ``NULL`` and the :c:type:`wchar_t*` string contains null characters a :exc:`ValueError` is raised. Returns a buffer allocated by :c:func:`PyMem_Alloc` (use - :c:func:`PyMem_Free` to free it) on success. On error, returns *NULL* + :c:func:`PyMem_Free` to free it) on success. On error, returns ``NULL`` and *\*size* is undefined. Raises a :exc:`MemoryError` if memory allocation is failed. .. versionadded:: 3.2 .. versionchanged:: 3.7 - Raises a :exc:`ValueError` if *size* is *NULL* and the :c:type:`wchar_t*` + Raises a :exc:`ValueError` if *size* is ``NULL`` and the :c:type:`wchar_t*` string contains null characters. @@ -982,15 +994,15 @@ Many of the following APIs take two arguments encoding and errors, and they have the same semantics as the ones of the built-in :func:`str` string object constructor. -Setting encoding to *NULL* causes the default encoding to be used -which is ASCII. The file system calls should use +Setting encoding to ``NULL`` causes the default encoding to be used +which is UTF-8. The file system calls should use :c:func:`PyUnicode_FSConverter` for encoding file names. This uses the variable :c:data:`Py_FileSystemDefaultEncoding` internally. This variable should be treated as read-only: on some systems, it will be a pointer to a static string, on others, it will change at run-time (such as when the application invokes setlocale). -Error handling is set by errors which may also be set to *NULL* meaning to use +Error handling is set by errors which may also be set to ``NULL`` meaning to use the default handling defined for the codec. Default error handling for all built-in codecs is "strict" (:exc:`ValueError` is raised). @@ -1010,7 +1022,7 @@ These are the generic codec APIs: Create a Unicode object by decoding *size* bytes of the encoded string *s*. *encoding* and *errors* have the same meaning as the parameters of the same name in the :func:`str` built-in function. The codec to be used is looked up - using the Python codec registry. Return *NULL* if an exception was raised by + using the Python codec registry. Return ``NULL`` if an exception was raised by the codec. @@ -1020,7 +1032,7 @@ These are the generic codec APIs: Encode a Unicode object and return the result as Python bytes object. *encoding* and *errors* have the same meaning as the parameters of the same name in the Unicode :meth:`~str.encode` method. The codec to be used is looked up - using the Python codec registry. Return *NULL* if an exception was raised by + using the Python codec registry. Return ``NULL`` if an exception was raised by the codec. @@ -1030,7 +1042,7 @@ These are the generic codec APIs: Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* and return a Python bytes object. *encoding* and *errors* have the same meaning as the parameters of the same name in the Unicode :meth:`~str.encode` method. The codec - to be used is looked up using the Python codec registry. Return *NULL* if an + to be used is looked up using the Python codec registry. Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 @@ -1047,14 +1059,14 @@ These are the UTF-8 codec APIs: .. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *s, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-8 encoded string - *s*. Return *NULL* if an exception was raised by the codec. + *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF8`. If - *consumed* is not *NULL*, trailing incomplete UTF-8 byte sequences will not be + If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF8`. If + *consumed* is not ``NULL``, trailing incomplete UTF-8 byte sequences will not be treated as an error. Those bytes will not be decoded and the number of bytes that have been decoded will be stored in *consumed*. @@ -1062,7 +1074,7 @@ These are the UTF-8 codec APIs: .. c:function:: PyObject* PyUnicode_AsUTF8String(PyObject *unicode) Encode a Unicode object using UTF-8 and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was + object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. @@ -1070,11 +1082,11 @@ These are the UTF-8 codec APIs: Return a pointer to the UTF-8 encoding of the Unicode object, and store the size of the encoded representation (in bytes) in *size*. The - *size* argument can be *NULL*; in this case no size will be stored. The + *size* argument can be ``NULL``; in this case no size will be stored. The returned buffer always has an extra null byte appended (not included in *size*), regardless of whether there are any other null code points. - In the case of an error, *NULL* is returned with an exception set and no + In the case of an error, ``NULL`` is returned with an exception set and no *size* is stored. This caches the UTF-8 representation of the string in the Unicode object, and @@ -1100,7 +1112,7 @@ These are the UTF-8 codec APIs: .. c:function:: PyObject* PyUnicode_EncodeUTF8(const Py_UNICODE *s, Py_ssize_t size, const char *errors) Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* using UTF-8 and - return a Python bytes object. Return *NULL* if an exception was raised by + return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 @@ -1119,10 +1131,10 @@ These are the UTF-32 codec APIs: const char *errors, int *byteorder) Decode *size* bytes from a UTF-32 encoded buffer string and return the - corresponding Unicode object. *errors* (if non-*NULL*) defines the error + corresponding Unicode object. *errors* (if non-``NULL``) defines the error handling. It defaults to "strict". - If *byteorder* is non-*NULL*, the decoder starts decoding using the given byte + If *byteorder* is non-``NULL``, the decoder starts decoding using the given byte order:: *byteorder == -1: little endian @@ -1137,16 +1149,16 @@ These are the UTF-32 codec APIs: After completion, *\*byteorder* is set to the current byte order at the end of input data. - If *byteorder* is *NULL*, the codec starts in native order mode. + If *byteorder* is ``NULL``, the codec starts in native order mode. - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF32`. If - *consumed* is not *NULL*, :c:func:`PyUnicode_DecodeUTF32Stateful` will not treat + If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF32`. If + *consumed* is not ``NULL``, :c:func:`PyUnicode_DecodeUTF32Stateful` will not treat trailing incomplete UTF-32 byte sequences (such as a number of bytes not divisible by four) as an error. Those bytes will not be decoded and the number of bytes that have been decoded will be stored in *consumed*. @@ -1156,7 +1168,7 @@ These are the UTF-32 codec APIs: Return a Python byte string using the UTF-32 encoding in native byte order. The string always starts with a BOM mark. Error handling is "strict". - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeUTF32(const Py_UNICODE *s, Py_ssize_t size, \ @@ -1172,10 +1184,10 @@ These are the UTF-32 codec APIs: If byteorder is ``0``, the output string will always start with the Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is prepended. - If *Py_UNICODE_WIDE* is not defined, surrogate pairs will be output + If ``Py_UNICODE_WIDE`` is not defined, surrogate pairs will be output as a single code point. - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using @@ -1192,10 +1204,10 @@ These are the UTF-16 codec APIs: const char *errors, int *byteorder) Decode *size* bytes from a UTF-16 encoded buffer string and return the - corresponding Unicode object. *errors* (if non-*NULL*) defines the error + corresponding Unicode object. *errors* (if non-``NULL``) defines the error handling. It defaults to "strict". - If *byteorder* is non-*NULL*, the decoder starts decoding using the given byte + If *byteorder* is non-``NULL``, the decoder starts decoding using the given byte order:: *byteorder == -1: little endian @@ -1211,16 +1223,16 @@ These are the UTF-16 codec APIs: After completion, *\*byteorder* is set to the current byte order at the end of input data. - If *byteorder* is *NULL*, the codec starts in native order mode. + If *byteorder* is ``NULL``, the codec starts in native order mode. - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF16`. If - *consumed* is not *NULL*, :c:func:`PyUnicode_DecodeUTF16Stateful` will not treat + If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF16`. If + *consumed* is not ``NULL``, :c:func:`PyUnicode_DecodeUTF16Stateful` will not treat trailing incomplete UTF-16 byte sequences (such as an odd number of bytes or a split surrogate pair) as an error. Those bytes will not be decoded and the number of bytes that have been decoded will be stored in *consumed*. @@ -1230,7 +1242,7 @@ These are the UTF-16 codec APIs: Return a Python byte string using the UTF-16 encoding in native byte order. The string always starts with a BOM mark. Error handling is "strict". - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeUTF16(const Py_UNICODE *s, Py_ssize_t size, \ @@ -1246,11 +1258,11 @@ These are the UTF-16 codec APIs: If byteorder is ``0``, the output string will always start with the Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is prepended. - If *Py_UNICODE_WIDE* is defined, a single :c:type:`Py_UNICODE` value may get + If ``Py_UNICODE_WIDE`` is defined, a single :c:type:`Py_UNICODE` value may get represented as a surrogate pair. If it is not defined, each :c:type:`Py_UNICODE` values is interpreted as a UCS-2 character. - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using @@ -1266,14 +1278,14 @@ These are the UTF-7 codec APIs: .. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *s, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-7 encoded string - *s*. Return *NULL* if an exception was raised by the codec. + *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF7`. If - *consumed* is not *NULL*, trailing incomplete UTF-7 base-64 sections will not + If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF7`. If + *consumed* is not ``NULL``, trailing incomplete UTF-7 base-64 sections will not be treated as an error. Those bytes will not be decoded and the number of bytes that have been decoded will be stored in *consumed*. @@ -1282,7 +1294,7 @@ These are the UTF-7 codec APIs: int base64SetO, int base64WhiteSpace, const char *errors) Encode the :c:type:`Py_UNICODE` buffer of the given size using UTF-7 and - return a Python bytes object. Return *NULL* if an exception was raised by + return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. If *base64SetO* is nonzero, "Set O" (punctuation that has no otherwise @@ -1305,20 +1317,20 @@ These are the "Unicode Escape" codec APIs: Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Unicode-Escape encoded - string *s*. Return *NULL* if an exception was raised by the codec. + string *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode) Encode a Unicode object using Unicode-Escape and return the result as a - bytes object. Error handling is "strict". Return *NULL* if an exception was + bytes object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, Py_ssize_t size) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Unicode-Escape and - return a bytes object. Return *NULL* if an exception was raised by the codec. + return a bytes object. Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using @@ -1335,13 +1347,13 @@ These are the "Raw Unicode Escape" codec APIs: Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Raw-Unicode-Escape - encoded string *s*. Return *NULL* if an exception was raised by the codec. + encoded string *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) Encode a Unicode object using Raw-Unicode-Escape and return the result as - a bytes object. Error handling is "strict". Return *NULL* if an exception + a bytes object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. @@ -1349,7 +1361,7 @@ These are the "Raw Unicode Escape" codec APIs: Py_ssize_t size) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape - and return a bytes object. Return *NULL* if an exception was raised by the codec. + and return a bytes object. Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using @@ -1367,20 +1379,20 @@ ordinals and only these are accepted by the codecs during encoding. .. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *s, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Latin-1 encoded string - *s*. Return *NULL* if an exception was raised by the codec. + *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode) Encode a Unicode object using Latin-1 and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was + object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeLatin1(const Py_UNICODE *s, Py_ssize_t size, const char *errors) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Latin-1 and - return a Python bytes object. Return *NULL* if an exception was raised by + return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 @@ -1399,20 +1411,20 @@ codes generate errors. .. c:function:: PyObject* PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the ASCII encoded string - *s*. Return *NULL* if an exception was raised by the codec. + *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode) Encode a Unicode object using ASCII and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was + object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeASCII(const Py_UNICODE *s, Py_ssize_t size, const char *errors) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using ASCII and - return a Python bytes object. Return *NULL* if an exception was raised by + return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 @@ -1436,10 +1448,10 @@ These are the mapping codec APIs: PyObject *mapping, const char *errors) Create a Unicode object by decoding *size* bytes of the encoded string *s* - using the given *mapping* object. Return *NULL* if an exception was raised + using the given *mapping* object. Return ``NULL`` if an exception was raised by the codec. - If *mapping* is *NULL*, Latin-1 decoding will be applied. Else + If *mapping* is ``NULL``, Latin-1 decoding will be applied. Else *mapping* must map bytes ordinals (integers in the range from 0 to 255) to Unicode strings, integers (which are then interpreted as Unicode ordinals) or ``None``. Unmapped data bytes -- ones which cause a @@ -1451,7 +1463,7 @@ These are the mapping codec APIs: .. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) Encode a Unicode object using the given *mapping* object and return the - result as a bytes object. Error handling is "strict". Return *NULL* if an + result as a bytes object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. The *mapping* object must map Unicode ordinal integers to bytes objects, @@ -1464,7 +1476,7 @@ These are the mapping codec APIs: PyObject *mapping, const char *errors) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given - *mapping* object and return the result as a bytes object. Return *NULL* if + *mapping* object and return the result as a bytes object. Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 @@ -1475,17 +1487,21 @@ These are the mapping codec APIs: The following codec API is special in that maps Unicode to Unicode. -.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \ - PyObject *mapping, const char *errors) +.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, const char *errors) - Translate a Unicode object using the given *mapping* object and return the - resulting Unicode object. Return *NULL* if an exception was raised by the + Translate a string by applying a character mapping table to it and return the + resulting Unicode object. Return ``NULL`` if an exception was raised by the codec. - The *mapping* object must map Unicode ordinal integers to Unicode strings, - integers (which are then interpreted as Unicode ordinals) or ``None`` - (causing deletion of the character). Unmapped character ordinals (ones - which cause a :exc:`LookupError`) are left untouched and are copied as-is. + The mapping table must map Unicode ordinal integers to Unicode ordinal integers + or ``None`` (causing deletion of the character). + + Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries + and sequences work well. Unmapped character ordinals (ones which cause a + :exc:`LookupError`) are left untouched and are copied as-is. + + *errors* has the usual meaning for codecs. It may be ``NULL`` which indicates to + use the default error handling. .. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ @@ -1493,7 +1509,7 @@ The following codec API is special in that maps Unicode to Unicode. Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a character *mapping* table to it and return the resulting Unicode object. - Return *NULL* when an exception was raised by the codec. + Return ``NULL`` when an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using @@ -1512,14 +1528,14 @@ the user settings on the machine running the codec. .. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *s, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the MBCS encoded string *s*. - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeMBCS`. If - *consumed* is not *NULL*, :c:func:`PyUnicode_DecodeMBCSStateful` will not decode + If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeMBCS`. If + *consumed* is not ``NULL``, :c:func:`PyUnicode_DecodeMBCSStateful` will not decode trailing lead byte and the number of bytes that have been decoded will be stored in *consumed*. @@ -1527,14 +1543,14 @@ the user settings on the machine running the codec. .. c:function:: PyObject* PyUnicode_AsMBCSString(PyObject *unicode) Encode a Unicode object using MBCS and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was + object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeCodePage(int code_page, PyObject *unicode, const char *errors) Encode the Unicode object using the specified code page and return a Python - bytes object. Return *NULL* if an exception was raised by the codec. Use + bytes object. Return ``NULL`` if an exception was raised by the codec. Use :c:data:`CP_ACP` code page to get the MBCS encoder. .. versionadded:: 3.3 @@ -1543,7 +1559,7 @@ the user settings on the machine running the codec. .. c:function:: PyObject* PyUnicode_EncodeMBCS(const Py_UNICODE *s, Py_ssize_t size, const char *errors) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using MBCS and return - a Python bytes object. Return *NULL* if an exception was raised by the + a Python bytes object. Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 @@ -1565,7 +1581,7 @@ The following APIs are capable of handling Unicode objects and strings on input (we refer to them as strings in the descriptions) and return Unicode objects or integers as appropriate. -They all return *NULL* or ``-1`` if an exception occurs. +They all return ``NULL`` or ``-1`` if an exception occurs. .. c:function:: PyObject* PyUnicode_Concat(PyObject *left, PyObject *right) @@ -1575,7 +1591,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: PyObject* PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) - Split a string giving a list of Unicode strings. If *sep* is *NULL*, splitting + Split a string giving a list of Unicode strings. If *sep* is ``NULL``, splitting will be done at all whitespace substrings. Otherwise, splits occur at the given separator. At most *maxsplit* splits will be done. If negative, no limit is set. Separators are not included in the resulting list. @@ -1588,23 +1604,6 @@ They all return *NULL* or ``-1`` if an exception occurs. characters are not included in the resulting strings. -.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, \ - const char *errors) - - Translate a string by applying a character mapping table to it and return the - resulting Unicode object. - - The mapping table must map Unicode ordinal integers to Unicode ordinal integers - or ``None`` (causing deletion of the character). - - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. - - *errors* has the usual meaning for codecs. It may be *NULL* which indicates to - use the default error handling. - - .. c:function:: PyObject* PyUnicode_Join(PyObject *separator, PyObject *seq) Join a sequence of strings using the given *separator* and return the resulting @@ -1670,7 +1669,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string) - Compare a unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less + Compare a Unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less than, equal, and greater than, respectively. It is best to pass only ASCII-encoded strings, but the function interprets the input string as ISO-8859-1 if it contains non-ASCII characters. @@ -1680,7 +1679,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) - Rich compare two unicode strings and return one of the following: + Rich compare two Unicode strings and return one of the following: * ``NULL`` in case an exception was raised * :const:`Py_True` or :const:`Py_False` for successful comparisons @@ -1708,7 +1707,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: void PyUnicode_InternInPlace(PyObject **string) Intern the argument *\*string* in place. The argument must be the address of a - pointer variable pointing to a Python unicode string object. If there is an + pointer variable pointing to a Python Unicode string object. If there is an existing interned string that is the same as *\*string*, it sets *\*string* to it (decrementing the reference count of the old string object and incrementing the reference count of the interned string object), otherwise it leaves @@ -1721,6 +1720,6 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: PyObject* PyUnicode_InternFromString(const char *v) A combination of :c:func:`PyUnicode_FromString` and - :c:func:`PyUnicode_InternInPlace`, returning either a new unicode string + :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string object that has been interned, or a new ("owned") reference to an earlier interned string object with the same value. diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst index d4484fb2712817..a805b564763c40 100644 --- a/Doc/c-api/utilities.rst +++ b/Doc/c-api/utilities.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _utilities: diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 317093e95615e3..b908cb8354f73f 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _veryhigh: @@ -16,11 +16,11 @@ parameter. The available start symbols are :const:`Py_eval_input`, :const:`Py_file_input`, and :const:`Py_single_input`. These are described following the functions which accept them as parameters. -Note also that several of these functions take :c:type:`FILE\*` parameters. One +Note also that several of these functions take :c:type:`FILE*` parameters. One particular issue which needs to be handled carefully is that the :c:type:`FILE` structure for different C libraries can be different and incompatible. Under Windows (at least), it is possible for dynamically linked extensions to actually -use different libraries, so care should be taken that :c:type:`FILE\*` parameters +use different libraries, so care should be taken that :c:type:`FILE*` parameters are only passed to these functions if it is certain that they were created by the same library that the Python runtime is using. @@ -42,10 +42,17 @@ the same library that the Python runtime is using. ``Py_InspectFlag`` is not set. +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings. + + .. versionadded:: 3.8 + + .. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving - *closeit* set to ``0`` and *flags* set to *NULL*. + *closeit* set to ``0`` and *flags* set to ``NULL``. .. c:function:: int PyRun_AnyFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) @@ -57,7 +64,7 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_AnyFileEx(FILE *fp, const char *filename, int closeit) This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving - the *flags* argument set to *NULL*. + the *flags* argument set to ``NULL``. .. c:function:: int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) @@ -66,14 +73,14 @@ the same library that the Python runtime is using. terminal input or Unix pseudo-terminal), return the value of :c:func:`PyRun_InteractiveLoop`, otherwise return the result of :c:func:`PyRun_SimpleFile`. *filename* is decoded from the filesystem - encoding (:func:`sys.getfilesystemencoding`). If *filename* is *NULL*, this + encoding (:func:`sys.getfilesystemencoding`). If *filename* is ``NULL``, this function uses ``"???"`` as the filename. .. c:function:: int PyRun_SimpleString(const char *command) This is a simplified interface to :c:func:`PyRun_SimpleStringFlags` below, - leaving the *PyCompilerFlags\** argument set to NULL. + leaving the :c:type:`PyCompilerFlags`\* argument set to ``NULL``. .. c:function:: int PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) @@ -92,13 +99,13 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_SimpleFile(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below, - leaving *closeit* set to ``0`` and *flags* set to *NULL*. + leaving *closeit* set to ``0`` and *flags* set to ``NULL``. .. c:function:: int PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit) This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below, - leaving *flags* set to *NULL*. + leaving *flags* set to ``NULL``. .. c:function:: int PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) @@ -110,14 +117,14 @@ the same library that the Python runtime is using. closed before PyRun_SimpleFileExFlags returns. .. note:: - On Windows, *fp* should be opened as binary mode (e.g. ``fopen(filename, "rb")``. + On Windows, *fp* should be opened as binary mode (e.g. ``fopen(filename, "rb")``). Otherwise, Python may not handle script file with LF line ending correctly. .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, - leaving *flags* set to *NULL*. + leaving *flags* set to ``NULL``. .. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) @@ -137,7 +144,7 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveLoopFlags` below, - leaving *flags* set to *NULL*. + leaving *flags* set to ``NULL``. .. c:function:: int PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) @@ -166,13 +173,13 @@ the same library that the Python runtime is using. ``char *func(FILE *stdin, FILE *stdout, char *prompt)``, overriding the default function used to read a single line of input at the interpreter's prompt. The function is expected to output - the string *prompt* if it's not *NULL*, and then read a line of + the string *prompt* if it's not ``NULL``, and then read a line of input from the provided standard input file, returning the resulting string. For example, The :mod:`readline` module sets this hook to provide line-editing and tab-completion features. The result must be a string allocated by :c:func:`PyMem_RawMalloc` or - :c:func:`PyMem_RawRealloc`, or *NULL* if an error occurred. + :c:func:`PyMem_RawRealloc`, or ``NULL`` if an error occurred. .. versionchanged:: 3.4 The result must be allocated by :c:func:`PyMem_RawMalloc` or @@ -184,14 +191,14 @@ the same library that the Python runtime is using. This is a simplified interface to :c:func:`PyParser_SimpleParseStringFlagsFilename` below, leaving *filename* set - to *NULL* and *flags* set to ``0``. + to ``NULL`` and *flags* set to ``0``. .. c:function:: struct _node* PyParser_SimpleParseStringFlags( const char *str, int start, int flags) This is a simplified interface to :c:func:`PyParser_SimpleParseStringFlagsFilename` below, leaving *filename* set - to *NULL*. + to ``NULL``. .. c:function:: struct _node* PyParser_SimpleParseStringFlagsFilename( const char *str, const char *filename, int start, int flags) @@ -218,7 +225,7 @@ the same library that the Python runtime is using. .. c:function:: PyObject* PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) This is a simplified interface to :c:func:`PyRun_StringFlags` below, leaving - *flags* set to *NULL*. + *flags* set to ``NULL``. .. c:function:: PyObject* PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) @@ -229,20 +236,20 @@ the same library that the Python runtime is using. that implements the mapping protocol. The parameter *start* specifies the start token that should be used to parse the source code. - Returns the result of executing the code as a Python object, or *NULL* if an + Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. .. c:function:: PyObject* PyRun_File(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals) This is a simplified interface to :c:func:`PyRun_FileExFlags` below, leaving - *closeit* set to ``0`` and *flags* set to *NULL*. + *closeit* set to ``0`` and *flags* set to ``NULL``. .. c:function:: PyObject* PyRun_FileEx(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, int closeit) This is a simplified interface to :c:func:`PyRun_FileExFlags` below, leaving - *flags* set to *NULL*. + *flags* set to ``NULL``. .. c:function:: PyObject* PyRun_FileFlags(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) @@ -263,7 +270,7 @@ the same library that the Python runtime is using. .. c:function:: PyObject* Py_CompileString(const char *str, const char *filename, int start) This is a simplified interface to :c:func:`Py_CompileStringFlags` below, leaving - *flags* set to *NULL*. + *flags* set to ``NULL``. .. c:function:: PyObject* Py_CompileStringFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags) @@ -279,7 +286,7 @@ the same library that the Python runtime is using. code which can be compiled and should be :const:`Py_eval_input`, :const:`Py_file_input`, or :const:`Py_single_input`. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or - :exc:`SyntaxError` exception messages. This returns *NULL* if the code + :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code cannot be parsed or compiled. The integer *optimize* specifies the optimization level of the compiler; a @@ -302,7 +309,7 @@ the same library that the Python runtime is using. This is a simplified interface to :c:func:`PyEval_EvalCodeEx`, with just the code object, and global and local variables. The other arguments are - set to *NULL*. + set to ``NULL``. .. c:function:: PyObject* PyEval_EvalCodeEx(PyObject *co, PyObject *globals, PyObject *locals, PyObject *const *args, int argcount, PyObject *const *kws, int kwcount, PyObject *const *defs, int defcount, PyObject *kwdefs, PyObject *closure) @@ -328,12 +335,12 @@ the same library that the Python runtime is using. .. c:function:: PyObject* PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) - This is the main, unvarnished function of Python interpretation. It is - literally 2000 lines long. The code object associated with the execution - frame *f* is executed, interpreting bytecode and executing calls as needed. - The additional *throwflag* parameter can mostly be ignored - if true, then - it causes an exception to immediately be thrown; this is used for the - :meth:`~generator.throw` methods of generator objects. + This is the main, unvarnished function of Python interpretation. The code + object associated with the execution frame *f* is executed, interpreting + bytecode and executing calls as needed. The additional *throwflag* + parameter can mostly be ignored - if true, then it causes an exception + to immediately be thrown; this is used for the :meth:`~generator.throw` + methods of generator objects. .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it @@ -379,13 +386,24 @@ the same library that the Python runtime is using. executed, it is passed as ``PyCompilerFlags *flags``. In this case, ``from __future__ import`` can modify *flags*. - Whenever ``PyCompilerFlags *flags`` is *NULL*, :attr:`cf_flags` is treated as + Whenever ``PyCompilerFlags *flags`` is ``NULL``, :attr:`cf_flags` is treated as equal to ``0``, and any modification due to ``from __future__ import`` is - discarded. :: + discarded. + + .. c:member:: int cf_flags + + Compiler flags. + + .. c:member:: int cf_feature_version + + *cf_feature_version* is the minor Python version. It should be + initialized to ``PY_MINOR_VERSION``. + + The field is ignored by default, it is used if and only if + ``PyCF_ONLY_AST`` flag is set in *cf_flags*. - struct PyCompilerFlags { - int cf_flags; - } + .. versionchanged:: 3.8 + Added *cf_feature_version* field. .. c:var:: int CO_FUTURE_DIVISION diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 6cb3e33fe843ac..e3a9bda54d671a 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _weakrefobjects: @@ -33,9 +33,9 @@ as much as it can. reference object may be returned. The second parameter, *callback*, can be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object - itself. *callback* may also be ``None`` or *NULL*. If *ob* is not a + itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly-referencable object, or if *callback* is not callable, ``None``, or - *NULL*, this will return *NULL* and raise :exc:`TypeError`. + ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -45,9 +45,9 @@ as much as it can. existing proxy object may be returned. The second parameter, *callback*, can be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak - reference object itself. *callback* may also be ``None`` or *NULL*. If *ob* + reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly-referencable object, or if *callback* is not callable, - ``None``, or *NULL*, this will return *NULL* and raise :exc:`TypeError`. + ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. .. c:function:: PyObject* PyWeakref_GetObject(PyObject *ref) diff --git a/Doc/conf.py b/Doc/conf.py index afe66270c10e8c..079d17717f381c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -14,7 +14,8 @@ # --------------------- extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', - 'pyspecific', 'c_annotations', 'escape4chm'] + 'pyspecific', 'c_annotations', 'escape4chm', + 'asdl_highlight', 'peg_highlight'] doctest_global_setup = ''' @@ -23,6 +24,9 @@ except ImportError: _tkinter = None ''' + +manpages_url = 'https://manpages.debian.org/{path}' + # General substitutions. project = 'Python' copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y') @@ -45,8 +49,10 @@ needs_sphinx = '1.8' # Ignore any .rst files in the venv/ directory. -venvdir = os.getenv('VENVDIR', 'venv') -exclude_patterns = [venvdir+'/*', 'README.rst'] +exclude_patterns = ['venv/*', 'README.rst'] +venvdir = os.getenv('VENVDIR') +if venvdir is not None: + exclude_patterns.append(venvdir + '/*') # Disable Docutils smartquotes for several translations smartquotes_excludes = { @@ -122,6 +128,7 @@ } \let\Verbatim=\OriginalVerbatim \let\endVerbatim=\endOriginalVerbatim +\setcounter{tocdepth}{2} ''' # The paper size ('letter' or 'a4'). @@ -221,3 +228,13 @@ # Relative filename of the reference count data file. refcount_file = 'data/refcounts.dat' + +# Sphinx 2 and Sphinx 3 compatibility +# ----------------------------------- + +# bpo-40204: Allow Sphinx 2 syntax in the C domain +c_allow_pre_v3 = True + +# bpo-40204: Disable warnings on Sphinx 2 syntax of the C domain since the +# documentation is built with -W (warnings treated as errors). +c_warn_on_allowed_pre_v3 = False diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 393a1f03751f82..1b90d9f172c992 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2019 Python Software Foundation. All rights reserved. +Copyright © 2001-2020 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 35527c179f3481..6b1bde37967ae9 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -234,6 +234,24 @@ PyCode_Check:PyObject*:co:0: PyCode_GetNumFree:int::: PyCode_GetNumFree:PyCodeObject*:co:0: +PyCode_NewWithPosOnlyArgs:PyCodeObject*::+1: +PyCode_NewWithPosOnlyArgs:int:argcount:: +PyCode_NewWithPosOnlyArgs:int:posonlyargcount:: +PyCode_NewWithPosOnlyArgs:int:kwonlyargcount:: +PyCode_NewWithPosOnlyArgs:int:nlocals:: +PyCode_NewWithPosOnlyArgs:int:stacksize:: +PyCode_NewWithPosOnlyArgs:int:flags:: +PyCode_NewWithPosOnlyArgs:PyObject*:code:0: +PyCode_NewWithPosOnlyArgs:PyObject*:consts:0: +PyCode_NewWithPosOnlyArgs:PyObject*:names:0: +PyCode_NewWithPosOnlyArgs:PyObject*:varnames:0: +PyCode_NewWithPosOnlyArgs:PyObject*:freevars:0: +PyCode_NewWithPosOnlyArgs:PyObject*:cellvars:0: +PyCode_NewWithPosOnlyArgs:PyObject*:filename:0: +PyCode_NewWithPosOnlyArgs:PyObject*:name:0: +PyCode_NewWithPosOnlyArgs:int:firstlineno:: +PyCode_NewWithPosOnlyArgs:PyObject*:lnotab:0: + PyCode_New:PyCodeObject*::+1: PyCode_New:int:argcount:: PyCode_New:int:kwonlyargcount:: @@ -413,6 +431,16 @@ PyDateTime_FromDateAndTime:int:minute:: PyDateTime_FromDateAndTime:int:second:: PyDateTime_FromDateAndTime:int:usecond:: +PyDateTime_FromDateAndTimeAndFold:PyObject*::+1: +PyDateTime_FromDateAndTimeAndFold:int:year:: +PyDateTime_FromDateAndTimeAndFold:int:month:: +PyDateTime_FromDateAndTimeAndFold:int:day:: +PyDateTime_FromDateAndTimeAndFold:int:hour:: +PyDateTime_FromDateAndTimeAndFold:int:minute:: +PyDateTime_FromDateAndTimeAndFold:int:second:: +PyDateTime_FromDateAndTimeAndFold:int:usecond:: +PyDateTime_FromDateAndTimeAndFold:int:fold:: + PyDateTime_FromTimestamp:PyObject*::+1: PyDateTime_FromTimestamp:PyObject*:args:0: @@ -931,6 +959,11 @@ PyGen_NewWithQualName:PyFrameObject*:frame:0: PyGen_NewWithQualName:PyObject*:name:0: PyGen_NewWithQualName:PyObject*:qualname:0: +PyGen_Send:int::: +PyGen_Send:PyGenObject*:gen:0: +PyGen_Send:PyObject*:arg:0: +PyGen_Send:PyObject**:presult:+1: + PyCoro_CheckExact:int::: PyCoro_CheckExact:PyObject*:ob:0: @@ -1177,11 +1210,6 @@ PyLong_FromString:const char*:str:: PyLong_FromString:char**:pend:: PyLong_FromString:int:base:: -PyLong_FromUnicode:PyObject*::+1: -PyLong_FromUnicode:Py_UNICODE*:u:: -PyLong_FromUnicode:Py_ssize_t:length:: -PyLong_FromUnicode:int:base:: - PyLong_FromUnicodeObject:PyObject*::+1: PyLong_FromUnicodeObject:PyObject*:u:0: PyLong_FromUnicodeObject:int:base:: @@ -1540,21 +1568,6 @@ PyOS_FSPath:PyObject*:path:0: PyObject_ASCII:PyObject*::+1: PyObject_ASCII:PyObject*:o:0: -PyObject_AsCharBuffer:int::: -PyObject_AsCharBuffer:PyObject*:obj:0: -PyObject_AsCharBuffer:const char**:buffer:: -PyObject_AsCharBuffer:Py_ssize_t*:buffer_len:: - -PyObject_AsReadBuffer:int::: -PyObject_AsReadBuffer:PyObject*:obj:0: -PyObject_AsReadBuffer:const void**:buffer:: -PyObject_AsReadBuffer:Py_ssize_t*:buffer_len:: - -PyObject_AsWriteBuffer:int::: -PyObject_AsWriteBuffer:PyObject*:obj:0: -PyObject_AsWriteBuffer:void**:buffer:: -PyObject_AsWriteBuffer:Py_ssize_t*:buffer_len:: - PyObject_Bytes:PyObject*::+1: PyObject_Bytes:PyObject*:o:0: @@ -1590,9 +1603,6 @@ PyObject_CallObject:PyObject*:args:0: PyObject_CheckBuffer:int::: PyObject_CheckBuffer:PyObject*:obj:0: -PyObject_CheckReadBuffer:int::: -PyObject_CheckReadBuffer:PyObject*:o:0: - PyObject_DelAttr:int::: PyObject_DelAttr:PyObject*:o:0: PyObject_DelAttr:PyObject*:attr_name:0: @@ -1645,7 +1655,8 @@ PyObject_GenericSetAttr:PyObject*:name:0: PyObject_GenericSetAttr:PyObject*:value:+1: PyObject_GenericSetDict:int::: -PyObject_GenericSetDict:PyObject*:o:+1: +PyObject_GenericSetDict:PyObject*:o:0: +PyObject_GenericSetDict:PyObject*:value:+1: PyObject_GenericSetDict:void*:context:: PyObject_GetAttr:PyObject*::+1: @@ -2210,6 +2221,13 @@ PyTime_FromTime:int:minute:: PyTime_FromTime:int:second:: PyTime_FromTime:int:usecond:: +PyTime_FromTimeAndFold:PyObject*::+1: +PyTime_FromTimeAndFold:int:hour:: +PyTime_FromTimeAndFold:int:minute:: +PyTime_FromTimeAndFold:int:second:: +PyTime_FromTimeAndFold:int:usecond:: +PyTime_FromTimeAndFold:int:fold:: + PyTraceMalloc_Track:int::: PyTraceMalloc_Track:unsigned int:domain:: PyTraceMalloc_Track:uintptr_t:ptr:: @@ -2270,6 +2288,11 @@ PyType_CheckExact:PyObject*:o:0: PyType_FromSpec:PyObject*::+1: PyType_FromSpec:PyType_Spec*:spec:: +PyType_FromModuleAndSpec:PyObject*::+1: +PyType_FromModuleAndSpec:PyObject*:module:+1: +PyType_FromModuleAndSpec:PyType_Spec*:spec:: +PyType_FromModuleAndSpec:PyObject*:bases:0: + PyType_FromSpecWithBases:PyObject*::+1: PyType_FromSpecWithBases:PyType_Spec*:spec:: PyType_FromSpecWithBases:PyObject*:bases:0: @@ -2406,9 +2429,6 @@ PyUnicode_AsUnicodeAndSize:Py_UNICODE*::: PyUnicode_AsUnicodeAndSize:PyObject*:unicode:0: PyUnicode_AsUnicodeAndSize:Py_ssize_t*:size:: -PyUnicode_AsUnicodeCopy:Py_UNICODE*::: -PyUnicode_AsUnicodeCopy:PyObject*:unicode:0: - PyUnicode_GetSize:Py_ssize_t::: PyUnicode_GetSize:PyObject*:unicode:0: @@ -2427,7 +2447,7 @@ PyUnicode_FromWideChar:Py_ssize_t:size:: PyUnicode_AsWideChar:Py_ssize_t::: PyUnicode_AsWideChar:PyObject*:*unicode:0: PyUnicode_AsWideChar:wchar_t*:w:: -PyUnicode_AsWideChar:Pyssize_t:size:: +PyUnicode_AsWideChar:Py_ssize_t:size:: PyUnicode_AsWideCharString:wchar_t*::: PyUnicode_AsWideCharString:PyObject*:unicode:0: @@ -3010,8 +3030,6 @@ Py_XINCREF:PyObject*:o:+1:if o is not NULL _PyImport_Fini:void::: -_PyImport_Init:void::: - _PyObject_New:PyObject*::+1: _PyObject_New:PyTypeObject*:type:0: diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 5dd14b1f7a60b9..02379946244d84 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -40,7 +40,7 @@ Key terms evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation and issue trackers on both `GitHub `__ and - `BitBucket `__. + `Bitbucket `__. * :mod:`distutils` is the original build and distribution system first added to the Python standard library in 1998. While direct use of :mod:`distutils` is being phased out, it still laid the foundation for the current packaging @@ -113,15 +113,22 @@ recommended tools`_. .. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations -Reading the guide -================= +.. index:: + single: Python Package Index (PyPI) + single: PyPI; (see Python Package Index (PyPI)) + +.. _publishing-python-packages: + +Reading the Python Packaging User Guide +======================================= The Python Packaging User Guide covers the various key steps and elements -involved in creating a project: +involved in creating and publishing a project: * `Project structure`_ * `Building and packaging the project`_ * `Uploading the project to the Python Packaging Index`_ +* `The .pypirc file`_ .. _Project structure: \ https://packaging.python.org/tutorials/distributing-packages/ @@ -129,6 +136,8 @@ involved in creating a project: https://packaging.python.org/tutorials/distributing-packages/#packaging-your-project .. _Uploading the project to the Python Packaging Index: \ https://packaging.python.org/tutorials/distributing-packages/#uploading-your-project-to-pypi +.. _The .pypirc file: \ + https://packaging.python.org/specifications/pypirc/ How do I...? @@ -142,7 +151,7 @@ These are quick answers or links for some common tasks. This isn't an easy topic, but here are a few tips: * check the Python Packaging Index to see if the name is already in use -* check popular hosting sites like GitHub, BitBucket, etc to see if there +* check popular hosting sites like GitHub, Bitbucket, etc to see if there is already a project with that name * check what comes up in a web search for the name you're considering * avoid particularly common words, especially ones with multiple meanings, diff --git a/Doc/distutils/_setuptools_disclaimer.rst b/Doc/distutils/_setuptools_disclaimer.rst new file mode 100644 index 00000000000000..cc75858326d44d --- /dev/null +++ b/Doc/distutils/_setuptools_disclaimer.rst @@ -0,0 +1,5 @@ +.. note:: + + This document is being retained solely until the ``setuptools`` documentation + at https://setuptools.readthedocs.io/en/latest/setuptools.html + independently covers all of the relevant information currently included here. diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index a825efc1a67243..b14197c2f94dba 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -4,6 +4,16 @@ API Reference ************* +.. seealso:: + + `New and changed setup.py arguments in setuptools`_ + The ``setuptools`` project adds new capabilities to the ``setup`` function + and other APIs, makes the API consistent across different Python versions, + and is hence recommended over using ``distutils`` directly. + +.. _New and changed setup.py arguments in setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords + +.. include:: ./_setuptools_disclaimer.rst :mod:`distutils.core` --- Core Distutils functionality ====================================================== @@ -277,6 +287,11 @@ the full reference. | | simply skip the extension. | | +------------------------+--------------------------------+---------------------------+ + .. versionchanged:: 3.8 + + On Unix, C extensions are no longer linked to libpython except on + Android and Cygwin. + .. class:: Distribution @@ -1127,6 +1142,24 @@ other utility module. * ``macosx-10.6-intel`` + For AIX, Python 3.9 and later return a string starting with "aix", followed + by additional fields (separated by ``'-'``) that represent the combined + values of AIX Version, Release and Technology Level (first field), Build Date + (second field), and bit-size (third field). Python 3.8 and earlier returned + only a single additional field with the AIX Version and Release. + + Examples of returned values on AIX: + + * ``aix-5307-0747-32`` # 32-bit build on AIX ``oslevel -s``: 5300-07-00-0000 + + * ``aix-7105-1731-64`` # 64-bit build on AIX ``oslevel -s``: 7100-05-01-1731 + + * ``aix-7.2`` # Legacy form reported in Python 3.8 and earlier + + .. versionchanged:: 3.9 + The AIX platform string format now also includes the technology level, + build date, and ABI bit-size. + .. function:: convert_path(pathname) @@ -1530,7 +1563,7 @@ Python's own build procedures. ================================================= .. module:: distutils.text_file - :synopsis: provides the TextFile class, a simple interface to text files + :synopsis: Provides the TextFile class, a simple interface to text files This module provides the :class:`TextFile` class, which gives an interface to @@ -1669,7 +1702,7 @@ lines, and joining lines with backslashes. =================================================== .. module:: distutils.version - :synopsis: implements classes that represent module version numbers. + :synopsis: Implements classes that represent module version numbers. .. % todo @@ -1684,7 +1717,7 @@ lines, and joining lines with backslashes. =================================================================== .. module:: distutils.cmd - :synopsis: This module provides the abstract base class Command. This class + :synopsis: Provides the abstract base class :class:`~distutils.cmd.Command`. This class is subclassed by the modules in the distutils.command subpackage. @@ -1777,7 +1810,7 @@ Subclasses of :class:`Command` must define the following methods. ========================================================== .. module:: distutils.command - :synopsis: This subpackage contains one module for each standard Distutils command. + :synopsis: Contains one module for each standard Distutils command. .. % \subsubsection{Individual Distutils commands} @@ -1822,6 +1855,9 @@ Subclasses of :class:`Command` must define the following methods. .. class:: bdist_msi +.. deprecated:: 3.9 + Use bdist_wheel (wheel packages) instead. + Builds a `Windows Installer`_ (.msi) binary package. .. _Windows Installer: https://msdn.microsoft.com/en-us/library/cc185688(VS.85).aspx @@ -1848,6 +1884,9 @@ Subclasses of :class:`Command` must define the following methods. .. module:: distutils.command.bdist_wininst :synopsis: Build a Windows installer +.. deprecated:: 3.8 + Use bdist_wheel (wheel packages) instead. + .. % todo @@ -2021,7 +2060,7 @@ This is described in more detail in :pep:`301`. =================================================================== .. module:: distutils.command.check - :synopsis: Check the metadata of a package + :synopsis: Check the meta-data of a package The ``check`` command performs some tests on the meta-data of a package. diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst index f1f3471261600e..e032c03e229a5c 100644 --- a/Doc/distutils/builtdist.rst +++ b/Doc/distutils/builtdist.rst @@ -4,6 +4,8 @@ Creating Built Distributions **************************** +.. include:: ./_setuptools_disclaimer.rst + A "built distribution" is what you're probably used to thinking of either as a "binary package" or an "installer" (depending on your background). It's not necessarily binary, though, because it might contain only Python source code @@ -144,6 +146,12 @@ generated by each, are: | :command:`bdist_msi` | msi | +--------------------------+-------------------------------------+ +.. note:: + bdist_wininst is deprecated since Python 3.8. + +.. note:: + bdist_msi is deprecated since Python 3.9. + The following sections give details on the individual :command:`bdist_\*` commands. @@ -296,6 +304,12 @@ file winds up deep in the "build tree," in a temporary directory created by Creating Windows Installers =========================== +.. warning:: + bdist_wininst is deprecated since Python 3.8. + +.. warning:: + bdist_msi is deprecated since Python 3.9. + Executable installers are the natural format for binary distributions on Windows. They display a nice graphical user interface, display some information about the module distribution to be installed taken from the metadata in the @@ -313,8 +327,8 @@ or the :command:`bdist` command with the :option:`!--formats` option:: If you have a pure module distribution (only containing pure Python modules and packages), the resulting installer will be version independent and have a name -like :file:`foo-1.0.win32.exe`. These installers can even be created on Unix -platforms or Mac OS X. +like :file:`foo-1.0.win32.exe`. Note that creating ``wininst`` binary +distributions in only supported on Windows systems. If you have a non-pure distribution, the extensions can only be created on a Windows platform, and will be Python version dependent. The installer filename @@ -457,3 +471,9 @@ Starting with Python 2.6, bdist_wininst supports a :option:`!--user-access-contr option. The default is 'none' (meaning no UAC handling is done), and other valid values are 'auto' (meaning prompt for UAC elevation if Python was installed for all users) and 'force' (meaning always prompt for elevation). + +.. note:: + bdist_wininst is deprecated since Python 3.8. + +.. note:: + bdist_msi is deprecated since Python 3.9. diff --git a/Doc/distutils/commandref.rst b/Doc/distutils/commandref.rst index 6a2ac960f1e1ef..0f6fe2aba865f8 100644 --- a/Doc/distutils/commandref.rst +++ b/Doc/distutils/commandref.rst @@ -4,6 +4,8 @@ Command Reference ***************** +.. include:: ./_setuptools_disclaimer.rst + .. % \section{Building modules: the \protect\command{build} command family} .. % \label{build-cmds} .. % \subsubsection{\protect\command{build}} diff --git a/Doc/distutils/configfile.rst b/Doc/distutils/configfile.rst index 0874d05fe703a1..2a5c8329e318b7 100644 --- a/Doc/distutils/configfile.rst +++ b/Doc/distutils/configfile.rst @@ -4,6 +4,8 @@ Writing the Setup Configuration File ************************************ +.. include:: ./_setuptools_disclaimer.rst + Often, it's not possible to write down everything needed to build a distribution *a priori*: you may need to get some information from the user, or from the user's system, in order to proceed. As long as that information is fairly diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst index f81e06b5e60578..e492b7f6057596 100644 --- a/Doc/distutils/examples.rst +++ b/Doc/distutils/examples.rst @@ -1,8 +1,10 @@ .. _distutils_examples: -******** -Examples -******** +****************** +Distutils Examples +****************** + +.. include:: ./_setuptools_disclaimer.rst This chapter provides a number of basic examples to help get started with distutils. Additional information about using distutils can be found in the @@ -253,7 +255,7 @@ Running the ``check`` command will display some warnings: running check warning: check: missing required meta-data: version, url warning: check: missing meta-data: either (author and author_email) or - (maintainer and maintainer_email) must be supplied + (maintainer and maintainer_email) should be supplied If you use the reStructuredText syntax in the ``long_description`` field and diff --git a/Doc/distutils/extending.rst b/Doc/distutils/extending.rst index 501fd7c564c6b9..1075e81779a7ba 100644 --- a/Doc/distutils/extending.rst +++ b/Doc/distutils/extending.rst @@ -4,6 +4,8 @@ Extending Distutils ******************* +.. include:: ./_setuptools_disclaimer.rst + Distutils can be extended in various ways. Most extensions take the form of new commands or replacements for existing commands. New commands may be written to support new types of platform-specific packaging, for example, while diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst index d6f7640fcb6826..1f72a25542494a 100644 --- a/Doc/distutils/index.rst +++ b/Doc/distutils/index.rst @@ -12,10 +12,7 @@ :ref:`distributing-index` The up to date module distribution documentations -This document describes the Python Distribution Utilities ("Distutils") from -the module developer's point of view, describing how to use the Distutils to -make Python modules and extensions easily available to a wider audience with -very little overhead for build/release/install mechanics. +.. include:: ./_setuptools_disclaimer.rst .. note:: @@ -25,6 +22,11 @@ very little overhead for build/release/install mechanics. recommendations section `__ in the Python Packaging User Guide for more information. +This document describes the Python Distribution Utilities ("Distutils") from +the module developer's point of view, describing the underlying capabilities +that ``setuptools`` builds on to allow Python developers to make Python modules +and extensions readily available to a wider audience. + .. toctree:: :maxdepth: 2 :numbered: @@ -34,7 +36,6 @@ very little overhead for build/release/install mechanics. configfile.rst sourcedist.rst builtdist.rst - packageindex.rst examples.rst extending.rst commandref.rst diff --git a/Doc/distutils/introduction.rst b/Doc/distutils/introduction.rst index 7721484fe73717..1f8a560e138616 100644 --- a/Doc/distutils/introduction.rst +++ b/Doc/distutils/introduction.rst @@ -4,6 +4,8 @@ An Introduction to Distutils **************************** +.. include:: ./_setuptools_disclaimer.rst + This document covers using the Distutils to distribute your Python modules, concentrating on the role of developer/distributor: if you're looking for information on installing Python modules, you should refer to the diff --git a/Doc/distutils/packageindex.rst b/Doc/distutils/packageindex.rst index 50cb74f2b6ca90..ccb9a598b2b7a2 100644 --- a/Doc/distutils/packageindex.rst +++ b/Doc/distutils/packageindex.rst @@ -1,6 +1,4 @@ -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) +:orphan: .. _package-index: @@ -8,246 +6,11 @@ The Python Package Index (PyPI) ******************************* -The `Python Package Index (PyPI)`_ stores :ref:`meta-data ` -describing distributions packaged with distutils, as well as package data like -distribution files if a package author wishes. - -Distutils provides the :command:`register` and :command:`upload` commands for -pushing meta-data and distribution files to PyPI, respectively. See -:ref:`package-commands` for information on these commands. - - -PyPI overview -============= - -PyPI lets you submit any number of versions of your distribution to the index. -If you alter the meta-data for a particular version, you can submit it again -and the index will be updated. - -PyPI holds a record for each (name, version) combination submitted. The first -user to submit information for a given name is designated the Owner of that -name. Changes can be submitted through the :command:`register` command or -through the web interface. Owners can designate other users as Owners or -Maintainers. Maintainers can edit the package information, but not designate -new Owners or Maintainers. - -By default PyPI displays only the newest version of a given package. The web -interface lets one change this default behavior and manually select which -versions to display and hide. - -For each version, PyPI displays a home page. The home page is created from -the ``long_description`` which can be submitted via the :command:`register` -command. See :ref:`package-display` for more information. - - -.. _package-commands: - -Distutils commands -================== - -Distutils exposes two commands for submitting package data to PyPI: the -:ref:`register ` command for submitting meta-data to PyPI -and the :ref:`upload ` command for submitting distribution -files. Both commands read configuration data from a special file called a -:ref:`.pypirc file `. - - -.. _package-register: - -The ``register`` command ------------------------- - -The distutils command :command:`register` is used to submit your distribution's -meta-data to an index server. It is invoked as follows:: - - python setup.py register - -Distutils will respond with the following prompt:: - - running register - We need to know who you are, so please choose either: - 1. use your existing login, - 2. register as a new user, - 3. have the server generate a new password for you (and email it to you), or - 4. quit - Your selection [default 1]: - -Note: if your username and password are saved locally, you will not see this -menu. Also, refer to :ref:`pypirc` for how to store your credentials in a -:file:`.pypirc` file. - -If you have not registered with PyPI, then you will need to do so now. You -should choose option 2, and enter your details as required. Soon after -submitting your details, you will receive an email which will be used to confirm -your registration. - -Once you are registered, you may choose option 1 from the menu. You will be -prompted for your PyPI username and password, and :command:`register` will then -submit your meta-data to the index. - -See :ref:`package-cmdoptions` for options to the :command:`register` command. - - -.. _package-upload: - -The ``upload`` command ----------------------- - -The distutils command :command:`upload` pushes the distribution files to PyPI. - -The command is invoked immediately after building one or more distribution -files. For example, the command :: - - python setup.py sdist bdist_wininst upload - -will cause the source distribution and the Windows installer to be uploaded to -PyPI. Note that these will be uploaded even if they are built using an earlier -invocation of :file:`setup.py`, but that only distributions named on the command -line for the invocation including the :command:`upload` command are uploaded. - -If a :command:`register` command was previously called in the same command, -and if the password was entered in the prompt, :command:`upload` will reuse the -entered password. This is useful if you do not want to store a password in -clear text in a :file:`.pypirc` file. - -You can use the ``--sign`` option to tell :command:`upload` to sign each -uploaded file using GPG (GNU Privacy Guard). The :program:`gpg` program must -be available for execution on the system :envvar:`PATH`. You can also specify -which key to use for signing using the ``--identity=name`` option. - -See :ref:`package-cmdoptions` for additional options to the :command:`upload` -command. - - -.. _package-cmdoptions: - -Additional command options --------------------------- - -This section describes options common to both the :command:`register` and -:command:`upload` commands. - -The ``--repository`` or ``-r`` option lets you specify a PyPI server -different from the default. For example:: - - python setup.py sdist bdist_wininst upload -r https://example.com/pypi - -For convenience, a name can be used in place of the URL when the -:file:`.pypirc` file is configured to do so. For example:: - - python setup.py register -r other - -See :ref:`pypirc` for more information on defining alternate servers. - -The ``--show-response`` option displays the full response text from the PyPI -server, which is useful when debugging problems with registering and uploading. - - -.. index:: - single: .pypirc file - single: Python Package Index (PyPI); .pypirc file - -.. _pypirc: - -The ``.pypirc`` file --------------------- - -The :command:`register` and :command:`upload` commands both check for the -existence of a :file:`.pypirc` file at the location :file:`$HOME/.pypirc`. -If this file exists, the command uses the username, password, and repository -URL configured in the file. The format of a :file:`.pypirc` file is as -follows: - -.. code-block:: ini - - [distutils] - index-servers = - pypi - - [pypi] - repository: - username: - password: - -The *distutils* section defines an *index-servers* variable that lists the -name of all sections describing a repository. - -Each section describing a repository defines three variables: - -- *repository*, that defines the url of the PyPI server. Defaults to - ``https://upload.pypi.org/legacy/``. -- *username*, which is the registered username on the PyPI server. -- *password*, that will be used to authenticate. If omitted the user - will be prompt to type it when needed. - -If you want to define another server a new section can be created and -listed in the *index-servers* variable: - -.. code-block:: ini - - [distutils] - index-servers = - pypi - other - - [pypi] - repository: - username: - password: - - [other] - repository: https://example.com/pypi - username: - password: - -This allows the :command:`register` and :command:`upload` commands to be -called with the ``--repository`` option as described in -:ref:`package-cmdoptions`. - -Specifically, you might want to add the `PyPI Test Repository -`_ to your ``.pypirc`` to facilitate -testing before doing your first upload to ``PyPI`` itself. - - -.. _package-display: - -PyPI package display -==================== - -The ``long_description`` field plays a special role at PyPI. It is used by -the server to display a home page for the registered package. - -If you use the `reStructuredText `_ -syntax for this field, PyPI will parse it and display an HTML output for -the package home page. - -The ``long_description`` field can be attached to a text file located -in the package:: - - from distutils.core import setup - - with open('README.txt') as file: - long_description = file.read() - - setup(name='Distutils', - long_description=long_description) - -In that case, :file:`README.txt` is a regular reStructuredText text file located -in the root of the package besides :file:`setup.py`. - -To prevent registering broken reStructuredText content, you can use the -:program:`rst2html` program that is provided by the :mod:`docutils` package and -check the ``long_description`` from the command line: - -.. code-block:: shell-session - - $ python setup.py --long-description | rst2html.py > output.html - -:mod:`docutils` will display a warning if there's something wrong with your -syntax. Because PyPI applies additional checks (e.g. by passing ``--no-raw`` -to ``rst2html.py`` in the command above), being able to run the command above -without warnings does not guarantee that PyPI will convert the content -successfully. +The `Python Package Index (PyPI)`_ stores metadata describing distributions +packaged with distutils and other publishing tools, as well the distribution +archives themselves. +References to up to date PyPI documentation can be found at +:ref:`publishing-python-packages`. .. _Python Package Index (PyPI): https://pypi.org diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst index 54ed1aebc242d8..4386a60b664bfb 100644 --- a/Doc/distutils/setupscript.rst +++ b/Doc/distutils/setupscript.rst @@ -4,6 +4,8 @@ Writing the Setup Script ************************ +.. include:: ./_setuptools_disclaimer.rst + The setup script is the centre of all activity in building, distributing, and installing modules using the Distutils. The main purpose of the setup script is to describe your module distribution to the Distutils, so that the various @@ -523,7 +525,7 @@ following way:: setup(..., data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']), - ('config', ['cfg/data.cfg']), + ('config', ['cfg/data.cfg'])], ) Each (*directory*, *files*) pair in the sequence specifies the installation @@ -612,9 +614,8 @@ Notes: provided, distutils lists it as the author in :file:`PKG-INFO`. (4) - The ``long_description`` field is used by PyPI when you are - :ref:`registering ` a package, to - :ref:`build its home page `. + The ``long_description`` field is used by PyPI when you publish a package, + to build its project page. (5) The ``license`` field is a text indicating the license covering the diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst index 0ac8ef41ddc20f..0600663d00e9dc 100644 --- a/Doc/distutils/sourcedist.rst +++ b/Doc/distutils/sourcedist.rst @@ -4,6 +4,8 @@ Creating a Source Distribution ****************************** +.. include:: ./_setuptools_disclaimer.rst + As shown in section :ref:`distutils-simple-example`, you use the :command:`sdist` command to create a source distribution. In the simplest case, :: diff --git a/Doc/distutils/uploading.rst b/Doc/distutils/uploading.rst index 4bce6997f9f83a..4c391cab072ea6 100644 --- a/Doc/distutils/uploading.rst +++ b/Doc/distutils/uploading.rst @@ -4,4 +4,5 @@ Uploading Packages to the Package Index *************************************** -The contents of this page have moved to the section :ref:`package-index`. +References to up to date PyPI documentation can be found at +:ref:`publishing-python-packages`. diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index 9bfad7fc3187d6..753b5511ed9d63 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _building: @@ -20,7 +20,7 @@ The initialization function has the signature: It returns either a fully-initialized module, or a :c:type:`PyModuleDef` instance. See :ref:`initializing-modules` for details. -.. highlightlang:: python +.. highlight:: python For modules with ASCII-only names, the function must be named ``PyInit_``, with ```` replaced by the name of the @@ -43,7 +43,7 @@ function corresponding to the filename is found. See the *"Multiple modules in one library"* section in :pep:`489` for details. -.. highlightlang:: c +.. highlight:: c Building C and C++ Extensions with distutils ============================================ diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 13d83b72f82ab6..5f5abdf9c15067 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _embedding: @@ -196,7 +196,7 @@ function is then made with:: pValue = PyObject_CallObject(pFunc, pArgs); -Upon return of the function, ``pValue`` is either *NULL* or it contains a +Upon return of the function, ``pValue`` is either ``NULL`` or it contains a reference to the return value of the function. Be sure to release the reference after examining the value. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 433178ab64d8e5..25dc2934d29ef6 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _extending-intro: @@ -117,7 +117,7 @@ store the converted values. More about this later. type and its components have been stored in the variables whose addresses are passed. It returns false (zero) if an invalid argument list was passed. In the latter case it also raises an appropriate exception so the calling function can -return *NULL* immediately (as we saw in the example). +return ``NULL`` immediately (as we saw in the example). .. _extending-errors: @@ -127,8 +127,8 @@ Intermezzo: Errors and Exceptions An important convention throughout the Python interpreter is the following: when a function fails, it should set an exception condition and return an error value -(usually a *NULL* pointer). Exceptions are stored in a static global variable -inside the interpreter; if this variable is *NULL* no exception has occurred. A +(usually a ``NULL`` pointer). Exceptions are stored in a static global variable +inside the interpreter; if this variable is ``NULL`` no exception has occurred. A second global variable stores the "associated value" of the exception (the second argument to :keyword:`raise`). A third variable contains the stack traceback in case the error originated in Python code. These three variables @@ -152,13 +152,13 @@ its associated value. You don't need to :c:func:`Py_INCREF` the objects passed to any of these functions. You can test non-destructively whether an exception has been set with -:c:func:`PyErr_Occurred`. This returns the current exception object, or *NULL* +:c:func:`PyErr_Occurred`. This returns the current exception object, or ``NULL`` if no exception has occurred. You normally don't need to call :c:func:`PyErr_Occurred` to see whether an error occurred in a function call, since you should be able to tell from the return value. When a function *f* that calls another function *g* detects that the latter -fails, *f* should itself return an error value (usually *NULL* or ``-1``). It +fails, *f* should itself return an error value (usually ``NULL`` or ``-1``). It should *not* call one of the :c:func:`PyErr_\*` functions --- one has already been called by *g*. *f*'s caller is then supposed to also return an error indication to *its* caller, again *without* calling :c:func:`PyErr_\*`, and so on @@ -209,7 +209,7 @@ usually declare a static object variable at the beginning of your file:: static PyObject *SpamError; and initialize it in your module's initialization function (:c:func:`PyInit_spam`) -with an exception object (leaving out the error checking for now):: +with an exception object:: PyMODINIT_FUNC PyInit_spam(void) @@ -221,14 +221,20 @@ with an exception object (leaving out the error checking for now):: return NULL; SpamError = PyErr_NewException("spam.error", NULL, NULL); - Py_INCREF(SpamError); - PyModule_AddObject(m, "error", SpamError); + Py_XINCREF(SpamError); + if (PyModule_AddObject(m, "error", SpamError) < 0) { + Py_XDECREF(SpamError); + Py_CLEAR(SpamError); + Py_DECREF(m); + return NULL; + } + return m; } Note that the Python name for the exception object is :exc:`spam.error`. The :c:func:`PyErr_NewException` function may create a class with the base class -being :exc:`Exception` (unless another class is passed in instead of *NULL*), +being :exc:`Exception` (unless another class is passed in instead of ``NULL``), described in :ref:`bltin-exceptions`. Note also that the :c:data:`SpamError` variable retains a reference to the newly @@ -272,7 +278,7 @@ statement:: if (!PyArg_ParseTuple(args, "s", &command)) return NULL; -It returns *NULL* (the error indicator for functions returning object pointers) +It returns ``NULL`` (the error indicator for functions returning object pointers) if an error is detected in the argument list, relying on the exception set by :c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been copied to the local variable :c:data:`command`. This is a pointer assignment and @@ -302,7 +308,7 @@ macro):: return Py_None; :c:data:`Py_None` is the C name for the special Python object ``None``. It is a -genuine Python object rather than a *NULL* pointer, which means "error" in most +genuine Python object rather than a ``NULL`` pointer, which means "error" in most contexts, as we have seen. @@ -370,7 +376,7 @@ inserts built-in function objects into the newly created module based upon the table (an array of :c:type:`PyMethodDef` structures) found in the module definition. :c:func:`PyModule_Create` returns a pointer to the module object that it creates. It may abort with a fatal error for -certain errors, or return *NULL* if the module could not be initialized +certain errors, or return ``NULL`` if the module could not be initialized satisfactorily. The init function must return the module object to its caller, so that it then gets inserted into ``sys.modules``. @@ -389,18 +395,26 @@ optionally followed by an import of the module:: } /* Add a built-in module, before Py_Initialize */ - PyImport_AppendInittab("spam", PyInit_spam); + if (PyImport_AppendInittab("spam", PyInit_spam) == -1) { + fprintf(stderr, "Error: could not extend in-built modules table\n"); + exit(1); + } /* Pass argv[0] to the Python interpreter */ Py_SetProgramName(program); - /* Initialize the Python interpreter. Required. */ + /* Initialize the Python interpreter. Required. + If this step fails, it will be a fatal error. */ Py_Initialize(); /* Optionally import the module; alternatively, import can be deferred until the embedded script imports it. */ - PyImport_ImportModule("spam"); + pmodule = PyImport_ImportModule("spam"); + if (!pmodule) { + PyErr_Print(); + fprintf(stderr, "Error: could not import module 'spam'\n"); + } ... @@ -520,8 +534,8 @@ This function must be registered with the interpreter using the :ref:`parsetuple`. The macros :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF` increment/decrement the -reference count of an object and are safe in the presence of *NULL* pointers -(but note that *temp* will not be *NULL* in this context). More info on them +reference count of an object and are safe in the presence of ``NULL`` pointers +(but note that *temp* will not be ``NULL`` in this context). More info on them in section :ref:`refcounts`. .. index:: single: PyObject_CallObject() @@ -530,7 +544,7 @@ Later, when it is time to call the function, you call the C function :c:func:`PyObject_CallObject`. This function has two arguments, both pointers to arbitrary Python objects: the Python function, and the argument list. The argument list must always be a tuple object, whose length is the number of -arguments. To call the Python function with no arguments, pass in NULL, or +arguments. To call the Python function with no arguments, pass in ``NULL``, or an empty tuple; to call it with one argument, pass a singleton tuple. :c:func:`Py_BuildValue` returns a tuple when its format string consists of zero or more format codes between parentheses. For example:: @@ -560,7 +574,7 @@ somehow :c:func:`Py_DECREF` the result, even (especially!) if you are not interested in its value. Before you do this, however, it is important to check that the return value -isn't *NULL*. If it is, the Python function terminated by raising an exception. +isn't ``NULL``. If it is, the Python function terminated by raising an exception. If the C code that called :c:func:`PyObject_CallObject` is called from Python, it should now return an error indication to its Python caller, so the interpreter can print a stack trace, or the calling Python code can handle the exception. @@ -717,7 +731,7 @@ The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: The *arg* and *format* parameters are identical to those of the :c:func:`PyArg_ParseTuple` function. The *kwdict* parameter is the dictionary of keywords received as the third parameter from the Python runtime. The *kwlist* -parameter is a *NULL*-terminated list of strings which identify the parameters; +parameter is a ``NULL``-terminated list of strings which identify the parameters; the names are matched with the type information from *format* from left to right. On success, :c:func:`PyArg_ParseTupleAndKeywords` returns true, otherwise it returns false and raises an appropriate exception. @@ -1078,32 +1092,32 @@ NULL Pointers ------------- In general, functions that take object references as arguments do not expect you -to pass them *NULL* pointers, and will dump core (or cause later core dumps) if -you do so. Functions that return object references generally return *NULL* only -to indicate that an exception occurred. The reason for not testing for *NULL* +to pass them ``NULL`` pointers, and will dump core (or cause later core dumps) if +you do so. Functions that return object references generally return ``NULL`` only +to indicate that an exception occurred. The reason for not testing for ``NULL`` arguments is that functions often pass the objects they receive on to other -function --- if each function were to test for *NULL*, there would be a lot of +function --- if each function were to test for ``NULL``, there would be a lot of redundant tests and the code would run more slowly. -It is better to test for *NULL* only at the "source:" when a pointer that may be -*NULL* is received, for example, from :c:func:`malloc` or from a function that +It is better to test for ``NULL`` only at the "source:" when a pointer that may be +``NULL`` is received, for example, from :c:func:`malloc` or from a function that may raise an exception. -The macros :c:func:`Py_INCREF` and :c:func:`Py_DECREF` do not check for *NULL* +The macros :c:func:`Py_INCREF` and :c:func:`Py_DECREF` do not check for ``NULL`` pointers --- however, their variants :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF` do. The macros for checking for a particular object type (``Pytype_Check()``) don't -check for *NULL* pointers --- again, there is much code that calls several of +check for ``NULL`` pointers --- again, there is much code that calls several of these in a row to test an object against various different expected types, and -this would generate redundant tests. There are no variants with *NULL* +this would generate redundant tests. There are no variants with ``NULL`` checking. The C function calling mechanism guarantees that the argument list passed to C -functions (``args`` in the examples) is never *NULL* --- in fact it guarantees +functions (``args`` in the examples) is never ``NULL`` --- in fact it guarantees that it is always a tuple [#]_. -It is a severe error to ever let a *NULL* pointer "escape" to the Python user. +It is a severe error to ever let a ``NULL`` pointer "escape" to the Python user. .. Frank Stajano: A pedagogically buggy example, along the lines of the previous listing, would @@ -1178,7 +1192,7 @@ different ways between the module providing the code and the client modules. Whichever method you choose, it's important to name your Capsules properly. The function :c:func:`PyCapsule_New` takes a name parameter -(:c:type:`const char \*`); you're permitted to pass in a *NULL* name, but +(:c:type:`const char \*`); you're permitted to pass in a ``NULL`` name, but we strongly encourage you to specify a name. Properly named Capsules provide a degree of runtime type-safety; there is no feasible way to tell one unnamed Capsule from another. @@ -1261,8 +1275,12 @@ function must take care of initializing the C API pointer array:: /* Create a Capsule containing the API pointer array's address */ c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL); - if (c_api_object != NULL) - PyModule_AddObject(m, "_C_API", c_api_object); + if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) { + Py_XDECREF(c_api_object); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 8b9549d7e39e2e..d9023709fddc86 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _new-types-topics: @@ -104,7 +104,7 @@ done. This can be done using the :c:func:`PyErr_Fetch` and /* This saves the current exception state */ PyErr_Fetch(&err_type, &err_value, &err_traceback); - cbresult = PyObject_CallObject(self->my_callback, NULL); + cbresult = PyObject_CallNoArgs(self->my_callback); if (cbresult == NULL) PyErr_WriteUnraisable(self->my_callback); else @@ -191,7 +191,7 @@ For every object which can support attributes, the corresponding type must provide the functions that control how the attributes are resolved. There needs to be a function which can retrieve attributes (if any are defined), and another to set attributes (if setting attributes is allowed). Removing an attribute is -a special case, for which the new value passed to the handler is *NULL*. +a special case, for which the new value passed to the handler is ``NULL``. Python supports two pairs of attribute handlers; a type that supports attributes only needs to implement the functions for one pair. The difference is that one @@ -233,9 +233,9 @@ attributes, when the values are computed, or how relevant data is stored. When :c:func:`PyType_Ready` is called, it uses three tables referenced by the type object to create :term:`descriptor`\s which are placed in the dictionary of the type object. Each descriptor controls access to one attribute of the instance -object. Each of the tables is optional; if all three are *NULL*, instances of +object. Each of the tables is optional; if all three are ``NULL``, instances of the type will only have attributes that are inherited from their base type, and -should leave the :c:member:`~PyTypeObject.tp_getattro` and :c:member:`~PyTypeObject.tp_setattro` fields *NULL* as +should leave the :c:member:`~PyTypeObject.tp_getattro` and :c:member:`~PyTypeObject.tp_setattro` fields ``NULL`` as well, allowing the base type to handle attributes. The tables are declared as three fields of the type object:: @@ -244,7 +244,7 @@ The tables are declared as three fields of the type object:: struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; -If :c:member:`~PyTypeObject.tp_methods` is not *NULL*, it must refer to an array of +If :c:member:`~PyTypeObject.tp_methods` is not ``NULL``, it must refer to an array of :c:type:`PyMethodDef` structures. Each entry in the table is an instance of this structure:: @@ -258,7 +258,7 @@ structure:: One entry should be defined for each method provided by the type; no entries are needed for methods inherited from a base type. One additional entry is needed at the end; it is a sentinel that marks the end of the array. The -:attr:`ml_name` field of the sentinel must be *NULL*. +:attr:`ml_name` field of the sentinel must be ``NULL``. The second table is used to define attributes which map directly to data stored in the instance. A variety of primitive C types are supported, and access may @@ -307,7 +307,7 @@ application can use the introspection API to retrieve the descriptor from the class object, and get the doc string using its :attr:`__doc__` attribute. As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :attr:`name` value -of *NULL* is required. +of ``NULL`` is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. @@ -352,9 +352,9 @@ Here is an example:: The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`__setattr__` or :meth:`__delattr__` method of a class instance would be called. When an -attribute should be deleted, the third parameter will be *NULL*. Here is an +attribute should be deleted, the third parameter will be ``NULL``. Here is an example that simply raises an exception; if this were really all you wanted, the -:c:member:`~PyTypeObject.tp_setattr` handler should be set to *NULL*. :: +:c:member:`~PyTypeObject.tp_setattr` handler should be set to ``NULL``. :: static int newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v) @@ -380,7 +380,7 @@ where the operator is one of ``Py_EQ``, ``Py_NE``, ``Py_LE``, ``Py_GT``, ``Py_LT`` or ``Py_GT``. It should compare the two objects with respect to the specified operator and return ``Py_True`` or ``Py_False`` if the comparison is successful, ``Py_NotImplemented`` to indicate that comparison is not -implemented and the other object's comparison method should be tried, or *NULL* +implemented and the other object's comparison method should be tried, or ``NULL`` if an exception was set. Here is a sample implementation, for a datatype that is considered equal if the @@ -427,7 +427,7 @@ from the type implementation, the older protocols have been defined as optional blocks of handlers referenced by the type object. For newer protocols there are additional slots in the main type object, with a flag bit being set to indicate that the slots are present and should be checked by the interpreter. (The flag -bit does not indicate that the slot values are non-*NULL*. The flag may be set +bit does not indicate that the slot values are non-``NULL``. The flag may be set to indicate the presence of a slot, but a slot may still be unfilled.) :: PyNumberMethods *tp_as_number; @@ -478,9 +478,9 @@ This function takes three arguments: :c:func:`PyArg_ParseTuple` to extract the arguments. #. *kwds* is a dictionary of keyword arguments that were passed. If this is - non-*NULL* and you support keyword arguments, use + non-``NULL`` and you support keyword arguments, use :c:func:`PyArg_ParseTupleAndKeywords` to extract the arguments. If you - do not want to support keyword arguments and this is non-*NULL*, raise a + do not want to support keyword arguments and this is non-``NULL``, raise a :exc:`TypeError` with a message saying that keyword arguments are not supported. Here is a toy ``tp_call`` implementation:: @@ -512,7 +512,7 @@ Here is a toy ``tp_call`` implementation:: These functions provide support for the iterator protocol. Both handlers take exactly one parameter, the instance for which they are being called, and return a new reference. In the case of an error, they should set an -exception and return *NULL*. :c:member:`~PyTypeObject.tp_iter` corresponds +exception and return ``NULL``. :c:member:`~PyTypeObject.tp_iter` corresponds to the Python :meth:`__iter__` method, while :c:member:`~PyTypeObject.tp_iternext` corresponds to the Python :meth:`~iterator.__next__` method. @@ -534,11 +534,11 @@ and :c:member:`~PyTypeObject.tp_iternext`. An iterator's to the iterator. Its :c:member:`~PyTypeObject.tp_iternext` handler should return a new reference to the next object in the iteration, if there is one. If the iteration has reached the end, :c:member:`~PyTypeObject.tp_iternext` -may return *NULL* without setting an exception, or it may set -:exc:`StopIteration` *in addition* to returning *NULL*; avoiding +may return ``NULL`` without setting an exception, or it may set +:exc:`StopIteration` *in addition* to returning ``NULL``; avoiding the exception can yield slightly better performance. If an actual error occurs, :c:member:`~PyTypeObject.tp_iternext` should always set an exception -and return *NULL*. +and return ``NULL``. .. _weakref-support: @@ -557,7 +557,7 @@ For an object to be weakly referencable, the extension type must do two things: #. Include a :c:type:`PyObject\*` field in the C object structure dedicated to the weak reference mechanism. The object's constructor should leave it - *NULL* (which is automatic when using the default + ``NULL`` (which is automatic when using the default :c:member:`~PyTypeObject.tp_alloc`). #. Set the :c:member:`~PyTypeObject.tp_weaklistoffset` type member @@ -582,7 +582,7 @@ And the corresponding member in the statically-declared type object:: The only further addition is that ``tp_dealloc`` needs to clear any weak references (by calling :c:func:`PyObject_ClearWeakRefs`) if the field is -non-*NULL*:: +non-``NULL``:: static void Trivial_dealloc(TrivialObject *self) diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index b4bf9b9e6f75f8..4da77e797d222c 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _defining-new-types: @@ -177,9 +177,14 @@ Everything else in the file should be familiar, except for some code in This initializes the :class:`Custom` type, filling in a number of members to the appropriate default values, including :attr:`ob_type` that we initially -set to *NULL*. :: +set to ``NULL``. :: - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + Py_INCREF(&CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } This adds the type to the module dictionary. This allows us to create :class:`Custom` instances by calling the :class:`Custom` class: @@ -270,7 +275,7 @@ which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: This method first clears the reference counts of the two Python attributes. :c:func:`Py_XDECREF` correctly handles the case where its argument is -*NULL* (which might happen here if ``tp_new`` failed midway). It then +``NULL`` (which might happen here if ``tp_new`` failed midway). It then calls the :c:member:`~PyTypeObject.tp_free` member of the object's type (computed by ``Py_TYPE(self)``) to free the object's memory. Note that the object's type might not be :class:`CustomType`, because the object may @@ -316,7 +321,7 @@ objects of the type. It is exposed in Python as the :meth:`__new__` method. It is not required to define a ``tp_new`` member, and indeed many extension types will simply reuse :c:func:`PyType_GenericNew` as done in the first version of the ``Custom`` type above. In this case, we use the ``tp_new`` -handler to initialize the ``first`` and ``last`` attributes to non-*NULL* +handler to initialize the ``first`` and ``last`` attributes to non-``NULL`` default values. ``tp_new`` is passed the type being instantiated (not necessarily ``CustomType``, @@ -336,7 +341,7 @@ slot to allocate memory:: self = (CustomObject *) type->tp_alloc(type, 0); Since memory allocation may fail, we must check the :c:member:`~PyTypeObject.tp_alloc` -result against *NULL* before proceeding. +result against ``NULL`` before proceeding. .. note:: We didn't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather @@ -411,7 +416,7 @@ But this would be risky. Our type doesn't restrict the type of the ``first`` member, so it could be any kind of object. It could have a destructor that causes code to be executed that tries to access the ``first`` member; or that destructor could release the -:term:`Global interpreter Lock` and let arbitrary code run in other +:term:`Global interpreter Lock ` and let arbitrary code run in other threads that accesses and modifies our object. To be paranoid and protect ourselves against this possibility, we almost @@ -450,9 +455,9 @@ below for details. A disadvantage of this approach is that it doesn't provide a way to restrict the types of objects that can be assigned to the Python attributes. We expect the first and last names to be strings, but any Python objects can be assigned. -Further, the attributes can be deleted, setting the C pointers to *NULL*. Even -though we can make sure the members are initialized to non-*NULL* values, the -members can be set to *NULL* if the attributes are deleted. +Further, the attributes can be deleted, setting the C pointers to ``NULL``. Even +though we can make sure the members are initialized to non-``NULL`` values, the +members can be set to ``NULL`` if the attributes are deleted. We define a single method, :meth:`Custom.name()`, that outputs the objects name as the concatenation of the first and last names. :: @@ -484,8 +489,8 @@ equivalent to the Python method: return "%s %s" % (self.first, self.last) Note that we have to check for the possibility that our :attr:`first` and -:attr:`last` members are *NULL*. This is because they can be deleted, in which -case they are set to *NULL*. It would be better to prevent deletion of these +:attr:`last` members are ``NULL``. This is because they can be deleted, in which +case they are set to ``NULL``. It would be better to prevent deletion of these attributes and to restrict the attribute values to be strings. We'll see how to do that in the next section. @@ -579,7 +584,7 @@ could, for example, be used to allow a single set of getter and setter functions that decide the attribute to get or set based on data in the closure.) The setter function is passed the :class:`Custom` object, the new value, and the -closure. The new value may be *NULL*, in which case the attribute is being +closure. The new value may be ``NULL``, in which case the attribute is being deleted. In our setter, we raise an error if the attribute is deleted or if its new value is not a string. @@ -598,7 +603,7 @@ and register it in the :c:member:`~PyTypeObject.tp_getset` slot:: .tp_getset = Custom_getsetters, The last item in a :c:type:`PyGetSetDef` structure is the "closure" mentioned -above. In this case, we aren't using a closure, so we just pass *NULL*. +above. In this case, we aren't using a closure, so we just pass ``NULL``. We also remove the member definitions for these attributes:: @@ -638,7 +643,7 @@ allow strings [#]_ to be passed:: } With these changes, we can assure that the ``first`` and ``last`` members are -never *NULL* so we can remove checks for *NULL* values in almost all cases. +never ``NULL`` so we can remove checks for ``NULL`` values in almost all cases. This means that most of the :c:func:`Py_XDECREF` calls can be converted to :c:func:`Py_DECREF` calls. The only place we can't change these calls is in the ``tp_dealloc`` implementation, where there is the possibility that the @@ -744,7 +749,7 @@ participate in cycles:: Notice the use of the :c:func:`Py_CLEAR` macro. It is the recommended and safe way to clear data attributes of arbitrary types while decrementing their reference counts. If you were to call :c:func:`Py_XDECREF` instead -on the attribute before setting it to *NULL*, there is a possibility +on the attribute before setting it to ``NULL``, there is a possibility that the attribute's destructor would call back into code that reads the attribute again (*especially* if there is a reference cycle). @@ -864,7 +869,12 @@ function:: return NULL; Py_INCREF(&SubListType); - PyModule_AddObject(m, "SubList", (PyObject *) &SubListType); + if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { + Py_DECREF(&SubListType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index 67bdd475aeb610..c7b92c6ea24ca8 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _building-on-windows: diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index e2d63a0323da66..8cf271c3024084 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -24,14 +24,16 @@ programmers will encounter a fragment of code like this:: z++; Only the ``x++`` statement is executed if the condition is true, but the -indentation leads you to believe otherwise. Even experienced C programmers will -sometimes stare at it a long time wondering why ``y`` is being decremented even +indentation leads many to believe otherwise. Even experienced C programmers will +sometimes stare at it a long time wondering as to why ``y`` is being decremented even for ``x > y``. Because there are no begin/end brackets, Python is much less prone to coding-style conflicts. In C there are many different ways to place the braces. -If you're used to reading and writing code that uses one style, you will feel at -least slightly uneasy when reading (or being required to write) another style. +After becoming used to reading and writing code using a particular style, +it is normal to feel somewhat uneasy when reading (or being required to write) +in a different one. + Many coding styles place begin/end brackets on a line by themselves. This makes programs considerably longer and wastes valuable screen space, making it harder @@ -146,69 +148,20 @@ variables and instance variables live in two different namespaces, and you need to tell Python which namespace to use. +.. _why-can-t-i-use-an-assignment-in-an-expression: + Why can't I use an assignment in an expression? ----------------------------------------------- -Many people used to C or Perl complain that they want to use this C idiom: - -.. code-block:: c - - while (line = readline(f)) { - // do something with line - } - -where in Python you're forced to write this:: - - while True: - line = f.readline() - if not line: - break - ... # do something with line - -The reason for not allowing assignment in Python expressions is a common, -hard-to-find bug in those other languages, caused by this construct: - -.. code-block:: c - - if (x = 0) { - // error handling - } - else { - // code that only works for nonzero x - } - -The error is a simple typo: ``x = 0``, which assigns 0 to the variable ``x``, -was written while the comparison ``x == 0`` is certainly what was intended. - -Many alternatives have been proposed. Most are hacks that save some typing but -use arbitrary or cryptic syntax or keywords, and fail the simple criterion for -language change proposals: it should intuitively suggest the proper meaning to a -human reader who has not yet been introduced to the construct. - -An interesting phenomenon is that most experienced Python programmers recognize -the ``while True`` idiom and don't seem to be missing the assignment in -expression construct much; it's only newcomers who express a strong desire to -add this to the language. - -There's an alternative way of spelling this that seems attractive but is -generally less robust than the "while True" solution:: - - line = f.readline() - while line: - ... # do something with line... - line = f.readline() +Starting in Python 3.8, you can! -The problem with this is that if you change your mind about exactly how you get -the next line (e.g. you want to change it into ``sys.stdin.readline()``) you -have to remember to change two places in your program -- the second occurrence -is hidden at the bottom of the loop. +Assignment expressions using the walrus operator `:=` assign a variable in an +expression:: -The best approach is to use iterators, making it possible to loop through -objects using the ``for`` statement. For example, :term:`file objects -` support the iterator protocol, so you can write simply:: + while chunk := fp.read(200): + print(chunk) - for line in f: - ... # do something with line... +See :pep:`572` for more information. @@ -620,8 +573,7 @@ whether an instance or a class implements a particular ABC. The :class:`~collections.abc.MutableMapping`. For Python, many of the advantages of interface specifications can be obtained -by an appropriate test discipline for components. There is also a tool, -PyChecker, which can be used to find problems due to subclassing. +by an appropriate test discipline for components. A good test suite for a module can both provide a regression test and serve as a module interface specification and a set of examples. Many Python modules can @@ -639,11 +591,11 @@ to the end of some internal list; an interface specification cannot test that your :meth:`append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. -Writing test suites is very helpful, and you might want to design your code with -an eye to making it easily tested. One increasingly popular technique, -test-directed development, calls for writing parts of the test suite first, -before you write any of the actual code. Of course Python allows you to be -sloppy and not write test cases at all. +Writing test suites is very helpful, and you might want to design your code to +make it easily tested. One increasingly popular technique, test-driven +development, calls for writing parts of the test suite first, before you write +any of the actual code. Of course Python allows you to be sloppy and not write +test cases at all. Why is there no goto? @@ -698,7 +650,7 @@ Why doesn't Python have a "with" statement for attribute assignments? --------------------------------------------------------------------- Python has a 'with' statement that wraps the execution of a block, calling code -on the entrance and exit from the block. Some language have a construct that +on the entrance and exit from the block. Some languages have a construct that looks like this:: with obj: diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 2ad27658705258..aecb56eaa4fd2f 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -89,7 +89,7 @@ For bytes, :c:func:`PyBytes_Size` returns its length and length. Note that Python bytes objects may contain null bytes so C's :c:func:`strlen` should not be used. -To test the type of an object, first make sure it isn't *NULL*, and then use +To test the type of an object, first make sure it isn't ``NULL``, and then use :c:func:`PyBytes_Check`, :c:func:`PyTuple_Check`, :c:func:`PyList_Check`, etc. There is also a high-level API to Python objects which is provided by the diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 3ef553e8acb43c..cf70f16c6fe328 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -17,12 +17,13 @@ What is Python? Python is an interpreted, interactive, object-oriented programming language. It incorporates modules, exceptions, dynamic typing, very high level dynamic data -types, and classes. Python combines remarkable power with very clear syntax. -It has interfaces to many system calls and libraries, as well as to various -window systems, and is extensible in C or C++. It is also usable as an -extension language for applications that need a programmable interface. -Finally, Python is portable: it runs on many Unix variants, on the Mac, and on -Windows 2000 and later. +types, and classes. It supports multiple programming paradigms beyond +object-oriented programming, such as procedural and functional programming. +Python combines remarkable power with very clear syntax. It has interfaces to +many system calls and libraries, as well as to various window systems, and is +extensible in C or C++. It is also usable as an extension language for +applications that need a programmable interface. Finally, Python is portable: +it runs on many Unix variants including Linux and macOS, and on Windows. To find out more, start with :ref:`tutorial-index`. The `Beginner's Guide to Python `_ links to other @@ -141,9 +142,9 @@ to fix critical bugs. Alpha, beta and release candidate versions have an additional suffix. The suffix for an alpha version is "aN" for some small number N, the suffix for a beta version is "bN" for some small number N, and the suffix for a release -candidate version is "cN" for some small number N. In other words, all versions +candidate version is "rcN" for some small number N. In other words, all versions labeled 2.0aN precede the versions labeled 2.0bN, which precede versions labeled -2.0cN, and *those* precede 2.0. +2.0rcN, and *those* precede 2.0. You may also find version numbers with a "+" suffix, e.g. "2.2+". These are unreleased versions, built directly from the CPython development repository. In @@ -295,8 +296,8 @@ How stable is Python? --------------------- Very stable. New, stable releases have been coming out roughly every 6 to 18 -months since 1991, and this seems likely to continue. Currently there are -usually around 18 months between major releases. +months since 1991, and this seems likely to continue. As of version 3.9, +Python will have a major new release every 12 months (:pep:`602`). The developers issue "bugfix" releases of older versions, so the stability of existing releases gradually improves. Bugfix releases, indicated by a third @@ -308,14 +309,14 @@ releases. The latest stable releases can always be found on the `Python download page `_. There are two production-ready versions of Python: 2.x and 3.x. The recommended version is 3.x, which is supported by -most widely used libraries. Although 2.x is still widely used, `it will not -be maintained after January 1, 2020 `_. +most widely used libraries. Although 2.x is still widely used, `it is not +maintained anymore `_. How many people are using Python? --------------------------------- -There are probably tens of thousands of users, though it's difficult to obtain -an exact count. +There are probably millions of users, though it's difficult to obtain an exact +count. Python is available for free download, so there are no sales figures, and it's available from many different sites and packaged with many Linux distributions, diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 4f9979bf55ed3a..781da467d18013 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -104,7 +104,7 @@ What platform-specific GUI toolkits exist for Python? ======================================================== By installing the `PyObjc Objective-C bridge -`_, Python programs can use Mac OS X's +`_, Python programs can use Mac OS X's Cocoa libraries. :ref:`Pythonwin ` by Mark Hammond includes an interface to the diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index ab92a879a88516..97058b5806a35c 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -125,7 +125,7 @@ argument list. It is called as :: handler(signum, frame) -so it should be declared with two arguments:: +so it should be declared with two parameters:: def handler(signum, frame): ... @@ -159,9 +159,9 @@ The "global main logic" of your program may be as simple as :: at the bottom of the main module of your program. -Once your program is organized as a tractable collection of functions and class -behaviours you should write test functions that exercise the behaviours. A test -suite that automates a sequence of tests can be associated with each module. +Once your program is organized as a tractable collection of function and class +behaviours, you should write test functions that exercise the behaviours. A +test suite that automates a sequence of tests can be associated with each module. This sounds like a lot of work, but since Python is so terse and flexible it's surprisingly easy. You can make coding much more pleasant and fun by writing your test functions in parallel with the "production code", since this makes it @@ -295,7 +295,7 @@ queue as there are threads. How do I parcel out work among a bunch of worker threads? --------------------------------------------------------- -The easiest way is to use the new :mod:`concurrent.futures` module, +The easiest way is to use the :mod:`concurrent.futures` module, especially the :mod:`~concurrent.futures.ThreadPoolExecutor` class. Or, if you want fine control over the dispatching algorithm, you can write @@ -679,7 +679,7 @@ How can I mimic CGI form submission (METHOD=POST)? I would like to retrieve web pages that are the result of POSTing a form. Is there existing code that would let me do this easily? -Yes. Here's a simple example that uses urllib.request:: +Yes. Here's a simple example that uses :mod:`urllib.request`:: #!/usr/local/bin/python @@ -765,20 +765,21 @@ The :mod:`select` module is commonly used to help with asynchronous I/O on sockets. To prevent the TCP connect from blocking, you can set the socket to non-blocking -mode. Then when you do the ``connect()``, you will either connect immediately +mode. Then when you do the :meth:`socket.connect`, you will either connect immediately (unlikely) or get an exception that contains the error number as ``.errno``. ``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn't finished yet. Different OSes will return different values, so you're going to have to check what's returned on your system. -You can use the ``connect_ex()`` method to avoid creating an exception. It will -just return the errno value. To poll, you can call ``connect_ex()`` again later +You can use the :meth:`socket.connect_ex` method to avoid creating an exception. It will +just return the errno value. To poll, you can call :meth:`socket.connect_ex` again later -- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can pass this -socket to select to check if it's writable. +socket to :meth:`select.select` to check if it's writable. .. note:: - The :mod:`asyncore` module presents a framework-like approach to the problem - of writing non-blocking networking code. + The :mod:`asyncio` module provides a general purpose single-threaded and + concurrent asynchronous library, which can be used for writing non-blocking + network code. The third-party `Twisted `_ library is a popular and feature-rich alternative. @@ -832,8 +833,8 @@ There are also many other specialized generators in this module, such as: Some higher-level functions operate on sequences directly, such as: -* ``choice(S)`` chooses random element from a given sequence -* ``shuffle(L)`` shuffles a list in-place, i.e. permutes it randomly +* ``choice(S)`` chooses a random element from a given sequence. +* ``shuffle(L)`` shuffles a list in-place, i.e. permutes it randomly. There's also a ``Random`` class you can instantiate to create independent multiple random number generators. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 31614189a62d2c..fd0adc378bfa6f 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -16,6 +16,9 @@ Is there a source code level debugger with breakpoints, single-stepping, etc.? Yes. +Several debuggers for Python are described below, and the built-in function +:func:`breakpoint` allows you to drop into any of them. + The pdb module is a simple but adequate console-mode debugger for Python. It is part of the standard Python library, and is :mod:`documented in the Library Reference Manual `. You can also write your own debugger by using the code @@ -32,12 +35,6 @@ for Windows Extensions `__ project an as a part of the ActivePython distribution (see https://www.activestate.com/activepython\ ). -`Boa Constructor `_ is an IDE and GUI -builder that uses wxWidgets. It offers visual frame creation and manipulation, -an object inspector, many views on the source like object browsers, inheritance -hierarchies, doc string generated html documentation, an advanced debugger, -integrated help, and Zope support. - `Eric `_ is an IDE built on PyQt and the Scintilla editing component. @@ -54,22 +51,14 @@ They include: * PyCharm (https://www.jetbrains.com/pycharm/) -Is there a tool to help find bugs or perform static analysis? +Are there tools to help find bugs or perform static analysis? ------------------------------------------------------------- Yes. -PyChecker is a static analysis tool that finds bugs in Python source code and -warns about code complexity and style. You can get PyChecker from -http://pychecker.sourceforge.net/. - -`Pylint `_ is another tool that checks -if a module satisfies a coding standard, and also makes it possible to write -plug-ins to add a custom feature. In addition to the bug checking that -PyChecker performs, Pylint offers some additional features such as checking line -length, whether variable names are well-formed according to your coding -standard, whether declared interfaces are fully implemented, and more. -https://docs.pylint.org/ provides a full list of Pylint's features. +`Pylint `_ and +`Pyflakes `_ do basic checking that will +help you catch bugs sooner. Static type checkers such as `Mypy `_, `Pyre `_, and @@ -515,14 +504,14 @@ desired effect in a number of ways. 1) By returning a tuple of the results:: - def func2(a, b): - a = 'new-value' # a and b are local names - b = b + 1 # assigned to new objects - return a, b # return new values - - x, y = 'old-value', 99 - x, y = func2(x, y) - print(x, y) # output: new-value 100 + >>> def func1(a, b): + ... a = 'new-value' # a and b are local names + ... b = b + 1 # assigned to new objects + ... return a, b # return new values + ... + >>> x, y = 'old-value', 99 + >>> func1(x, y) + ('new-value', 100) This is almost always the clearest solution. @@ -530,38 +519,41 @@ desired effect in a number of ways. 3) By passing a mutable (changeable in-place) object:: - def func1(a): - a[0] = 'new-value' # 'a' references a mutable list - a[1] = a[1] + 1 # changes a shared object - - args = ['old-value', 99] - func1(args) - print(args[0], args[1]) # output: new-value 100 + >>> def func2(a): + ... a[0] = 'new-value' # 'a' references a mutable list + ... a[1] = a[1] + 1 # changes a shared object + ... + >>> args = ['old-value', 99] + >>> func2(args) + >>> args + ['new-value', 100] 4) By passing in a dictionary that gets mutated:: - def func3(args): - args['a'] = 'new-value' # args is a mutable dictionary - args['b'] = args['b'] + 1 # change it in-place - - args = {'a': 'old-value', 'b': 99} - func3(args) - print(args['a'], args['b']) + >>> def func3(args): + ... args['a'] = 'new-value' # args is a mutable dictionary + ... args['b'] = args['b'] + 1 # change it in-place + ... + >>> args = {'a': 'old-value', 'b': 99} + >>> func3(args) + >>> args + {'a': 'new-value', 'b': 100} 5) Or bundle up values in a class instance:: - class callByRef: - def __init__(self, **args): - for (key, value) in args.items(): - setattr(self, key, value) - - def func4(args): - args.a = 'new-value' # args is a mutable callByRef - args.b = args.b + 1 # change object in-place - - args = callByRef(a='old-value', b=99) - func4(args) - print(args.a, args.b) + >>> class Namespace: + ... def __init__(self, /, **args): + ... for key, value in args.items(): + ... setattr(self, key, value) + ... + >>> def func4(args): + ... args.a = 'new-value' # args is a mutable Namespace + ... args.b = args.b + 1 # change object in-place + ... + >>> args = Namespace(a='old-value', b=99) + >>> func4(args) + >>> vars(args) + {'a': 'new-value', 'b': 100} There's almost never a good reason to get this complicated. @@ -656,7 +648,7 @@ How can my code discover the name of an object? ----------------------------------------------- Generally speaking, it can't, because objects don't really have names. -Essentially, assignment always binds a name to a value; The same is true of +Essentially, assignment always binds a name to a value; the same is true of ``def`` and ``class`` statements, but in that case the value is a callable. Consider the following code:: @@ -776,30 +768,23 @@ A slash in the argument list of a function denotes that the parameters prior to it are positional-only. Positional-only parameters are the ones without an externally-usable name. Upon calling a function that accepts positional-only parameters, arguments are mapped to parameters based solely on their position. -For example, :func:`pow` is a function that accepts positional-only parameters. -Its documentation looks like this:: - - >>> help(pow) - Help on built-in function pow in module builtins: +For example, :func:`divmod` is a function that accepts positional-only +parameters. Its documentation looks like this:: - pow(x, y, z=None, /) - Equivalent to x**y (with two arguments) or x**y % z (with three arguments) + >>> help(divmod) + Help on built-in function divmod in module builtins: - Some types, such as ints, are able to use a more efficient algorithm when - invoked using the three argument form. + divmod(x, y, /) + Return the tuple (x//y, x%y). Invariant: div*y + mod == x. -The slash at the end of the parameter list means that all three parameters are -positional-only. Thus, calling :func:`pow` with keyword aguments would lead to -an error:: +The slash at the end of the parameter list means that both parameters are +positional-only. Thus, calling :func:`divmod` with keyword arguments would lead +to an error:: - >>> pow(x=3, y=4) + >>> divmod(x=3, y=4) Traceback (most recent call last): File "", line 1, in - TypeError: pow() takes no keyword arguments - -Note that as of this writing this is only documentational and no valid syntax -in Python, although there is :pep:`570`, which proposes a syntax for -position-only parameters in Python. + TypeError: divmod() takes no keyword arguments Numbers and strings @@ -855,10 +840,11 @@ For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') e.g. ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == -144`` and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, base)`` takes -the base to convert from as a second optional argument, so ``int('0x144', 16) == -324``. If the base is specified as 0, the number is interpreted using Python's -rules: a leading '0o' indicates octal, and '0x' indicates a hex number. +144`` holds true, and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, +base)`` takes the base to convert from as a second optional argument, so ``int( +'0x144', 16) == 324``. If the base is specified as 0, the number is interpreted +using Python's rules: a leading '0o' indicates octal, and '0x' indicates a hex +number. Do not use the built-in function :func:`eval` if all you need is to convert strings to numbers. :func:`eval` will be significantly slower and it presents a @@ -1023,7 +1009,7 @@ That's a tough one, in general. First, here are a list of things to remember before diving further: * Performance characteristics vary across Python implementations. This FAQ - focusses on :term:`CPython`. + focuses on :term:`CPython`. * Behaviour can vary across operating systems, especially when talking about I/O or multi-threading. * You should always find the hot spots in your program *before* attempting to @@ -1155,7 +1141,7 @@ How do you remove duplicates from a list? See the Python Cookbook for a long discussion of many ways to do this: - https://code.activestate.com/recipes/52560/ + https://github.com/ActiveState/code/tree/master/recipes/Python/52560_Remove_duplicates/recipe-52560.py If you don't mind reordering the list, sort it and then scan from the end of the list, deleting duplicates as you go:: @@ -1498,8 +1484,8 @@ to uppercase:: Here the ``UpperOut`` class redefines the ``write()`` method to convert the argument string to uppercase before calling the underlying -``self.__outfile.write()`` method. All other methods are delegated to the -underlying ``self.__outfile`` object. The delegation is accomplished via the +``self._outfile.write()`` method. All other methods are delegated to the +underlying ``self._outfile`` object. The delegation is accomplished via the ``__getattr__`` method; consult :ref:`the language reference ` for more information about controlling attribute access. @@ -1518,20 +1504,19 @@ Most :meth:`__setattr__` implementations must modify ``self.__dict__`` to store local state for self without causing an infinite recursion. -How do I call a method defined in a base class from a derived class that overrides it? --------------------------------------------------------------------------------------- +How do I call a method defined in a base class from a derived class that extends it? +------------------------------------------------------------------------------------ Use the built-in :func:`super` function:: class Derived(Base): def meth(self): - super(Derived, self).meth() + super().meth() # calls Base.meth -For version prior to 3.0, you may be using classic classes: For a class -definition such as ``class Derived(Base): ...`` you can call method ``meth()`` -defined in ``Base`` (or one of ``Base``'s base classes) as ``Base.meth(self, -arguments...)``. Here, ``Base.meth`` is an unbound method, so you need to -provide the ``self`` argument. +In the example, :func:`super` will automatically determine the instance from +which it was called (the ``self`` value), look up the :term:`method resolution +order` (MRO) with ``type(self).__mro__``, and return the next in line after +``Derived`` in the MRO: ``Base``. How can I organize my code to make it easier to change the base class? diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index ad97cd0932ac86..a181086e9ce627 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -1,6 +1,6 @@ :tocdepth: 2 -.. highlightlang:: none +.. highlight:: none .. _windows-faq: diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 5810a6b7997804..7be755e4113108 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -15,10 +15,10 @@ Glossary ``...`` Can refer to: - * The default Python prompt of the interactive shell when entering code for - an indented code block, when within a pair of matching left and right - delimiters (parentheses, square brackets, curly braces or triple quotes), - or after specifying a decorator. + * The default Python prompt of the interactive shell when entering the + code for an indented code block, when within a pair of matching left and + right delimiters (parentheses, square brackets, curly braces or triple + quotes), or after specifying a decorator. * The :const:`Ellipsis` built-in constant. @@ -189,6 +189,10 @@ Glossary A list of bytecode instructions can be found in the documentation for :ref:`the dis module `. + callback + A subroutine function which is passed as an argument to be executed at + some point in the future. + class A template for creating user-defined objects. Class definitions normally contain method definitions which operate on instances of the @@ -225,6 +229,15 @@ Glossary statement by defining :meth:`__enter__` and :meth:`__exit__` methods. See :pep:`343`. + context variable + A variable which can have different values depending on its context. + This is similar to Thread-Local Storage in which each execution + thread may have a different value for a variable. However, with context + variables, there may be several contexts in one execution thread and the + main usage for context variables is to keep track of variables in + concurrent asynchronous tasks. + See :mod:`contextvars`. + contiguous .. index:: C-contiguous, Fortran contiguous @@ -238,7 +251,7 @@ Glossary Fortran contiguous arrays, the first index varies the fastest. coroutine - Coroutines is a more generalized form of subroutines. Subroutines are + Coroutines are a more generalized form of subroutines. Subroutines are entered at one point and exited at another point. Coroutines can be entered, exited, and resumed at many different points. They can be implemented with the :keyword:`async def` statement. See also @@ -503,8 +516,10 @@ Glossary Hashability makes an object usable as a dictionary key and a set member, because these data structures use the hash value internally. - All of Python's immutable built-in objects are hashable; mutable - containers (such as lists or dictionaries) are not. Objects which are + Most of Python's immutable built-in objects are hashable; mutable + containers (such as lists or dictionaries) are not; immutable + containers (such as tuples and frozensets) are only hashable if + their elements are hashable. Objects which are instances of user-defined classes are hashable by default. They all compare unequal (except with themselves), and their hash value is derived from their :func:`id`. @@ -572,7 +587,7 @@ Glossary and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects `, and objects of any classes you define with an :meth:`__iter__` method or with a :meth:`__getitem__` method - that implements :term:`Sequence` semantics. + that implements :term:`Sequence ` semantics. Iterables can be used in a :keyword:`for` loop and in many other places where a sequence is @@ -728,17 +743,28 @@ Glossary also :term:`immutable`. named tuple - Any tuple-like class whose indexable elements are also accessible using - named attributes (for example, :func:`time.localtime` returns a - tuple-like object where the *year* is accessible either with an - index such as ``t[0]`` or with a named attribute like ``t.tm_year``). - - A named tuple can be a built-in type such as :class:`time.struct_time`, - or it can be created with a regular class definition. A full featured - named tuple can also be created with the factory function - :func:`collections.namedtuple`. The latter approach automatically - provides extra features such as a self-documenting representation like - ``Employee(name='jones', title='programmer')``. + The term "named tuple" applies to any type or class that inherits from + tuple and whose indexable elements are also accessible using named + attributes. The type or class may have other features as well. + + Several built-in types are named tuples, including the values returned + by :func:`time.localtime` and :func:`os.stat`. Another example is + :data:`sys.float_info`:: + + >>> sys.float_info[1] # indexed access + 1024 + >>> sys.float_info.max_exp # named field access + 1024 + >>> isinstance(sys.float_info, tuple) # kind of tuple + True + + Some named tuples are built-in types (such as the above examples). + Alternatively, a named tuple can be created from a regular class + definition that inherits from :class:`tuple` and that defines named + fields. Such a class can be written by hand or it can be created with + the factory function :func:`collections.namedtuple`. The latter + technique also adds some extra methods that may not be found in + hand-written or built-in named tuples. namespace The place where a variable is stored. Namespaces are implemented as @@ -802,9 +828,11 @@ Glossary .. _positional-only_parameter: * :dfn:`positional-only`: specifies an argument that can be supplied only - by position. Python has no syntax for defining positional-only - parameters. However, some built-in functions have positional-only - parameters (e.g. :func:`abs`). + by position. Positional-only parameters can be defined by including a + ``/`` character in the parameter list of the function definition after + them, for example *posonly1* and *posonly2* in the following:: + + def func(posonly1, posonly2, /, positional_or_keyword): ... .. _keyword-only_parameter: @@ -1021,14 +1049,6 @@ Glossary an :term:`expression` or one of several constructs with a keyword, such as :keyword:`if`, :keyword:`while` or :keyword:`for`. - struct sequence - A tuple with named elements. Struct sequences expose an interface similar - to :term:`named tuple` in that elements can be accessed either by - index or as an attribute. However, they do not have any of the named tuple - methods like :meth:`~collections.somenamedtuple._make` or - :meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences - include :data:`sys.float_info` and the return value of :func:`os.stat`. - text encoding A codec which encodes Unicode strings to bytes. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index e78a022b372faa..76d8e6be42935f 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -353,7 +353,7 @@ Our program keeps growing in complexity:: args = parser.parse_args() answer = args.square**2 if args.verbose: - print("the square of {} equals {}".format(args.square, answer)) + print(f"the square of {args.square} equals {answer}") else: print(answer) @@ -387,9 +387,9 @@ multiple verbosity values, and actually get to use them:: args = parser.parse_args() answer = args.square**2 if args.verbosity == 2: - print("the square of {} equals {}".format(args.square, answer)) + print(f"the square of {args.square} equals {answer}") elif args.verbosity == 1: - print("{}^2 == {}".format(args.square, answer)) + print(f"{args.square}^2 == {answer}") else: print(answer) @@ -421,9 +421,9 @@ Let's fix it by restricting the values the ``--verbosity`` option can accept:: args = parser.parse_args() answer = args.square**2 if args.verbosity == 2: - print("the square of {} equals {}".format(args.square, answer)) + print(f"the square of {args.square} equals {answer}") elif args.verbosity == 1: - print("{}^2 == {}".format(args.square, answer)) + print(f"{args.square}^2 == {answer}") else: print(answer) @@ -461,9 +461,9 @@ verbosity argument (check the output of ``python --help``):: args = parser.parse_args() answer = args.square**2 if args.verbosity == 2: - print("the square of {} equals {}".format(args.square, answer)) + print(f"the square of {args.square} equals {answer}") elif args.verbosity == 1: - print("{}^2 == {}".format(args.square, answer)) + print(f"{args.square}^2 == {answer}") else: print(answer) @@ -529,9 +529,9 @@ Let's fix:: # bugfix: replace == with >= if args.verbosity >= 2: - print("the square of {} equals {}".format(args.square, answer)) + print(f"the square of {args.square} equals {answer}") elif args.verbosity >= 1: - print("{}^2 == {}".format(args.square, answer)) + print(f"{args.square}^2 == {answer}") else: print(answer) @@ -566,9 +566,9 @@ Let's fix that bug:: args = parser.parse_args() answer = args.square**2 if args.verbosity >= 2: - print("the square of {} equals {}".format(args.square, answer)) + print(f"the square of {args.square} equals {answer}") elif args.verbosity >= 1: - print("{}^2 == {}".format(args.square, answer)) + print(f"{args.square}^2 == {answer}") else: print(answer) @@ -606,9 +606,9 @@ not just squares:: args = parser.parse_args() answer = args.x**args.y if args.verbosity >= 2: - print("{} to the power {} equals {}".format(args.x, args.y, answer)) + print(f"{args.x} to the power {args.y} equals {answer}") elif args.verbosity >= 1: - print("{}^{} == {}".format(args.x, args.y, answer)) + print(f"{args.x}^{args.y} == {answer}") else: print(answer) @@ -645,9 +645,9 @@ to display *more* text instead:: args = parser.parse_args() answer = args.x**args.y if args.verbosity >= 2: - print("Running '{}'".format(__file__)) + print(f"Running '{__file__}'") if args.verbosity >= 1: - print("{}^{} == ".format(args.x, args.y), end="") + print(f"{args.x}^{args.y} == ", end="") print(answer) Output: @@ -688,9 +688,9 @@ which will be the opposite of the ``--verbose`` one:: if args.quiet: print(answer) elif args.verbose: - print("{} to the power {} equals {}".format(args.x, args.y, answer)) + print(f"{args.x} to the power {args.y} equals {answer}") else: - print("{}^{} == {}".format(args.x, args.y, answer)) + print(f"{args.x}^{args.y} == {answer}") Our program is now simpler, and we've lost some functionality for the sake of demonstration. Anyways, here's the output: diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 5b2457a1682bc6..50041829b8c388 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c ********************** Argument Clinic How-To @@ -1070,7 +1070,7 @@ Currently Argument Clinic supports only a few return converters: DecodeFSDefault None of these take parameters. For the first three, return -1 to indicate -error. For ``DecodeFSDefault``, the return type is ``const char *``; return a NULL +error. For ``DecodeFSDefault``, the return type is ``const char *``; return a ``NULL`` pointer to indicate an error. (There's also an experimental ``NoneType`` converter, which lets you diff --git a/Doc/howto/cporting.rst b/Doc/howto/cporting.rst index b638e32f5d0ecb..ce7700fc599062 100644 --- a/Doc/howto/cporting.rst +++ b/Doc/howto/cporting.rst @@ -1,4 +1,4 @@ -.. highlightlang:: c +.. highlight:: c .. _cporting-howto: diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index b29e590b20cba8..b792b6c6ab77f2 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -117,7 +117,7 @@ The important points to remember are: * non-data descriptors may be overridden by instance dictionaries. The object returned by ``super()`` also has a custom :meth:`__getattribute__` -method for invoking descriptors. The call ``super(B, obj).m()`` searches +method for invoking descriptors. The attribute lookup ``super(B, obj).m`` searches ``obj.__class__.__mro__`` for the base class ``A`` immediately following ``B`` and then returns ``A.__dict__['m'].__get__(obj, B)``. If not a descriptor, ``m`` is returned unchanged. If not in the dictionary, ``m`` reverts to a @@ -145,7 +145,7 @@ print a message for each get or set. Overriding :meth:`__getattribute__` is alternate approach that could do this for every attribute. However, this descriptor is useful for monitoring just a few chosen attributes:: - class RevealAccess(object): + class RevealAccess: """A data descriptor that sets and returns values normally and prints a message logging their access. """ @@ -162,7 +162,7 @@ descriptor is useful for monitoring just a few chosen attributes:: print('Updating', self.name) self.val = val - >>> class MyClass(object): + >>> class MyClass: ... x = RevealAccess(10, 'var "x"') ... y = 5 ... @@ -194,7 +194,7 @@ triggers function calls upon access to an attribute. Its signature is:: The documentation shows a typical use to define a managed attribute ``x``:: - class C(object): + class C: def getx(self): return self.__x def setx(self, value): self.__x = value def delx(self): del self.__x @@ -203,7 +203,7 @@ The documentation shows a typical use to define a managed attribute ``x``:: To see how :func:`property` is implemented in terms of the descriptor protocol, here is a pure Python equivalent:: - class Property(object): + class Property: "Emulate PyProperty_Type() in Objects/descrobject.c" def __init__(self, fget=None, fset=None, fdel=None, doc=None): @@ -250,7 +250,7 @@ to be recalculated on every access; however, the programmer does not want to affect existing client code accessing the attribute directly. The solution is to wrap access to the value attribute in a property data descriptor:: - class Cell(object): + class Cell: . . . def getvalue(self): "Recalculate the cell before returning value" @@ -277,7 +277,7 @@ binding methods during attribute access. This means that all functions are non-data descriptors which return bound methods when they are invoked from an object. In pure Python, it works like this:: - class Function(object): + class Function: . . . def __get__(self, obj, objtype=None): "Simulate func_descr_get() in Objects/funcobject.c" @@ -287,7 +287,7 @@ object. In pure Python, it works like this:: Running the interpreter shows how the function descriptor works in practice:: - >>> class D(object): + >>> class D: ... def f(self, x): ... return x ... @@ -312,14 +312,12 @@ Running the interpreter shows how the function descriptor works in practice:: >>> d.f > - # Internally, the bound method stores the underlying function, - # the bound instance, and the class of the bound instance. + # Internally, the bound method stores the underlying function and + # the bound instance. >>> d.f.__func__ >>> d.f.__self__ <__main__.D object at 0x1012e1f98> - >>> d.f.__class__ - Static Methods and Class Methods @@ -367,7 +365,7 @@ It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` o Since staticmethods return the underlying function with no changes, the example calls are unexciting:: - >>> class E(object): + >>> class E: ... def f(x): ... print(x) ... f = staticmethod(f) @@ -380,7 +378,7 @@ calls are unexciting:: Using the non-data descriptor protocol, a pure Python version of :func:`staticmethod` would look like this:: - class StaticMethod(object): + class StaticMethod: "Emulate PyStaticMethod_Type() in Objects/funcobject.c" def __init__(self, f): @@ -393,7 +391,7 @@ Unlike static methods, class methods prepend the class reference to the argument list before calling the function. This format is the same for whether the caller is an object or a class:: - >>> class E(object): + >>> class E: ... def f(klass, x): ... return klass.__name__, x ... f = classmethod(f) @@ -410,7 +408,7 @@ is to create alternate class constructors. In Python 2.3, the classmethod :func:`dict.fromkeys` creates a new dictionary from a list of keys. The pure Python equivalent is:: - class Dict(object): + class Dict: . . . def fromkeys(klass, iterable, value=None): "Emulate dict_fromkeys() in Objects/dictobject.c" @@ -428,7 +426,7 @@ Now a new dictionary of unique keys can be constructed like this:: Using the non-data descriptor protocol, a pure Python version of :func:`classmethod` would look like this:: - class ClassMethod(object): + class ClassMethod: "Emulate PyClassMethod_Type() in Objects/funcobject.c" def __init__(self, f): diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index f8f2aac70f9b06..74e861480d2ff8 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1229,9 +1229,9 @@ Text Processing". Mertz also wrote a 3-part series of articles on functional programming for IBM's DeveloperWorks site; see -`part 1 `__, -`part 2 `__, and -`part 3 `__, +`part 1 `__, +`part 2 `__, and +`part 3 `__, Python documentation diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 50cde3595034b5..f0081e4ec28905 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -272,9 +272,7 @@ should instead read: Available static markers ------------------------ -.. I'm reusing the "c:function" type for markers - -.. c:function:: function__entry(str filename, str funcname, int lineno) +.. object:: function__entry(str filename, str funcname, int lineno) This marker indicates that execution of a Python function has begun. It is only triggered for pure-Python (bytecode) functions. @@ -290,7 +288,7 @@ Available static markers * ``$arg3`` : ``int`` line number -.. c:function:: function__return(str filename, str funcname, int lineno) +.. object:: function__return(str filename, str funcname, int lineno) This marker is the converse of :c:func:`function__entry`, and indicates that execution of a Python function has ended (either via ``return``, or via an @@ -298,7 +296,7 @@ Available static markers The arguments are the same as for :c:func:`function__entry` -.. c:function:: line(str filename, str funcname, int lineno) +.. object:: line(str filename, str funcname, int lineno) This marker indicates a Python line is about to be executed. It is the equivalent of line-by-line tracing with a Python profiler. It is @@ -306,24 +304,24 @@ Available static markers The arguments are the same as for :c:func:`function__entry`. -.. c:function:: gc__start(int generation) +.. object:: gc__start(int generation) Fires when the Python interpreter starts a garbage collection cycle. ``arg0`` is the generation to scan, like :func:`gc.collect()`. -.. c:function:: gc__done(long collected) +.. object:: gc__done(long collected) Fires when the Python interpreter finishes a garbage collection cycle. ``arg0`` is the number of collected objects. -.. c:function:: import__find__load__start(str modulename) +.. object:: import__find__load__start(str modulename) Fires before :mod:`importlib` attempts to find and load the module. ``arg0`` is the module name. .. versionadded:: 3.7 -.. c:function:: import__find__load__done(str modulename, int found) +.. object:: import__find__load__done(str modulename, int found) Fires after :mod:`importlib`'s find_and_load function is called. ``arg0`` is the module name, ``arg1`` indicates if module was @@ -332,6 +330,15 @@ Available static markers .. versionadded:: 3.7 +.. object:: audit(str event, void *tuple) + + Fires when :func:`sys.audit` or :c:func:`PySys_Audit` is called. + ``arg0`` is the event name as C string, ``arg1`` is a :c:type:`PyObject` + pointer to a tuple object. + + .. versionadded:: 3.8 + + SystemTap Tapsets ----------------- @@ -366,14 +373,14 @@ If this file is installed in SystemTap's tapset directory (e.g. ``/usr/share/systemtap/tapset``), then these additional probepoints become available: -.. c:function:: python.function.entry(str filename, str funcname, int lineno, frameptr) +.. object:: python.function.entry(str filename, str funcname, int lineno, frameptr) This probe point indicates that execution of a Python function has begun. It is only triggered for pure-Python (bytecode) functions. -.. c:function:: python.function.return(str filename, str funcname, int lineno, frameptr) +.. object:: python.function.return(str filename, str funcname, int lineno, frameptr) - This probe point is the converse of :c:func:`python.function.return`, and + This probe point is the converse of ``python.function.return``, and indicates that execution of a Python function has ended (either via ``return``, or via an exception). It is only triggered for pure-Python (bytecode) functions. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 4956aa0dd957bc..de0f834551f5dd 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -579,7 +579,7 @@ information. When you call one of the logging methods on an instance of information in the delegated call. Here's a snippet from the code of :class:`LoggerAdapter`:: - def debug(self, msg, *args, **kwargs): + def debug(self, msg, /, *args, **kwargs): """ Delegate a debug call to the underlying logger, after adding contextual information from this adapter instance. @@ -948,6 +948,41 @@ This variant shows how you can e.g. apply configuration for particular loggers machinery in the main process (even though the logging events are generated in the worker processes) to direct the messages to the appropriate destinations. +Using concurrent.futures.ProcessPoolExecutor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want to use :class:`concurrent.futures.ProcessPoolExecutor` to start +your worker processes, you need to create the queue slightly differently. +Instead of + +.. code-block:: python + + queue = multiprocessing.Queue(-1) + +you should use + +.. code-block:: python + + queue = multiprocessing.Manager().Queue(-1) # also works with the examples above + +and you can then replace the worker creation from this:: + + workers = [] + for i in range(10): + worker = multiprocessing.Process(target=worker_process, + args=(queue, worker_configurer)) + workers.append(worker) + worker.start() + for w in workers: + w.join() + +to this (remembering to first import :mod:`concurrent.futures`):: + + with concurrent.futures.ProcessPoolExecutor(max_workers=10) as executor: + for i in range(10): + executor.submit(worker_process, queue, worker_configurer) + + Using file rotation ------------------- @@ -1079,7 +1114,7 @@ call ``str()`` on that object to get the actual format string. Consider the following two classes:: class BraceMessage: - def __init__(self, fmt, *args, **kwargs): + def __init__(self, fmt, /, *args, **kwargs): self.fmt = fmt self.args = args self.kwargs = kwargs @@ -1088,7 +1123,7 @@ following two classes:: return self.fmt.format(*self.args, **self.kwargs) class DollarMessage: - def __init__(self, fmt, **kwargs): + def __init__(self, fmt, /, **kwargs): self.fmt = fmt self.kwargs = kwargs @@ -1143,7 +1178,7 @@ to the above, as in the following example:: import logging - class Message(object): + class Message: def __init__(self, fmt, args): self.fmt = fmt self.args = args @@ -1155,7 +1190,7 @@ to the above, as in the following example:: def __init__(self, logger, extra=None): super(StyleAdapter, self).__init__(logger, extra or {}) - def log(self, level, msg, *args, **kwargs): + def log(self, level, msg, /, *args, **kwargs): if self.isEnabledFor(level): msg, kwargs = self.process(msg, kwargs) self.logger._log(level, Message(msg, args), (), **kwargs) @@ -1301,7 +1336,7 @@ You can also subclass :class:`QueueListener` to get messages from other kinds of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: class ZeroMQSocketListener(QueueListener): - def __init__(self, uri, *handlers, **kwargs): + def __init__(self, uri, /, *handlers, **kwargs): self.ctx = kwargs.get('ctx') or zmq.Context() socket = zmq.Socket(self.ctx, zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, '') # subscribe to everything @@ -1333,7 +1368,7 @@ An example dictionary-based configuration ----------------------------------------- Below is an example of a logging configuration dictionary - it's taken from -the `documentation on the Django project `_. +the `documentation on the Django project `_. This dictionary is passed to :func:`~config.dictConfig` to put the configuration into effect:: LOGGING = { @@ -1389,7 +1424,7 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration } For more information about this configuration, you can see the `relevant -section `_ +section `_ of the Django documentation. .. _cookbook-rotator-namer: @@ -1706,8 +1741,8 @@ which uses JSON to serialise the event in a machine-parseable manner:: import json import logging - class StructuredMessage(object): - def __init__(self, message, **kwargs): + class StructuredMessage: + def __init__(self, message, /, **kwargs): self.message = message self.kwargs = kwargs @@ -1750,8 +1785,8 @@ as in the following complete example:: return o.encode('unicode_escape').decode('ascii') return super(Encoder, self).default(o) - class StructuredMessage(object): - def __init__(self, message, **kwargs): + class StructuredMessage: + def __init__(self, message, /, **kwargs): self.message = message self.kwargs = kwargs @@ -1982,8 +2017,8 @@ object as a message format string, and that the logging package will call :func:`str` on that object to get the actual format string. Consider the following two classes:: - class BraceMessage(object): - def __init__(self, fmt, *args, **kwargs): + class BraceMessage: + def __init__(self, fmt, /, *args, **kwargs): self.fmt = fmt self.args = args self.kwargs = kwargs @@ -1991,8 +2026,8 @@ following two classes:: def __str__(self): return self.fmt.format(*self.args, **self.kwargs) - class DollarMessage(object): - def __init__(self, fmt, **kwargs): + class DollarMessage: + def __init__(self, fmt, /, **kwargs): self.fmt = fmt self.kwargs = kwargs @@ -2266,9 +2301,9 @@ The script just arranges to decorate ``foo`` with a decorator which will do the conditional logging that's required. The decorator takes a logger as a parameter and attaches a memory handler for the duration of the call to the decorated function. The decorator can be additionally parameterised using a target handler, -a level at which flushing should occur, and a capacity for the buffer. These -default to a :class:`~logging.StreamHandler` which writes to ``sys.stderr``, -``logging.ERROR`` and ``100`` respectively. +a level at which flushing should occur, and a capacity for the buffer (number of +records buffered). These default to a :class:`~logging.StreamHandler` which +writes to ``sys.stderr``, ``logging.ERROR`` and ``100`` respectively. Here's the script:: @@ -2457,7 +2492,7 @@ scope of the context manager:: import logging import sys - class LoggingContext(object): + class LoggingContext: def __init__(self, logger, level=None, handler=None, close=True): self.logger = logger self.level = level @@ -2709,3 +2744,238 @@ And if we want less: In this case, the commands don't print anything to the console, since nothing at ``WARNING`` level or above is logged by them. + +.. _qt-gui: + +A Qt GUI for logging +-------------------- + +A question that comes up from time to time is about how to log to a GUI +application. The `Qt `_ framework is a popular +cross-platform UI framework with Python bindings using `PySide2 +`_ or `PyQt5 +`_ libraries. + +The following example shows how to log to a Qt GUI. This introduces a simple +``QtHandler`` class which takes a callable, which should be a slot in the main +thread that does GUI updates. A worker thread is also created to show how you +can log to the GUI from both the UI itself (via a button for manual logging) +as well as a worker thread doing work in the background (here, just logging +messages at random levels with random short delays in between). + +The worker thread is implemented using Qt's ``QThread`` class rather than the +:mod:`threading` module, as there are circumstances where one has to use +``QThread``, which offers better integration with other ``Qt`` components. + +The code should work with recent releases of either ``PySide2`` or ``PyQt5``. +You should be able to adapt the approach to earlier versions of Qt. Please +refer to the comments in the code snippet for more detailed information. + +.. code-block:: python3 + + import datetime + import logging + import random + import sys + import time + + # Deal with minor differences between PySide2 and PyQt5 + try: + from PySide2 import QtCore, QtGui, QtWidgets + Signal = QtCore.Signal + Slot = QtCore.Slot + except ImportError: + from PyQt5 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot + + + logger = logging.getLogger(__name__) + + + # + # Signals need to be contained in a QObject or subclass in order to be correctly + # initialized. + # + class Signaller(QtCore.QObject): + signal = Signal(str, logging.LogRecord) + + # + # Output to a Qt GUI is only supposed to happen on the main thread. So, this + # handler is designed to take a slot function which is set up to run in the main + # thread. In this example, the function takes a string argument which is a + # formatted log message, and the log record which generated it. The formatted + # string is just a convenience - you could format a string for output any way + # you like in the slot function itself. + # + # You specify the slot function to do whatever GUI updates you want. The handler + # doesn't know or care about specific UI elements. + # + class QtHandler(logging.Handler): + def __init__(self, slotfunc, *args, **kwargs): + super(QtHandler, self).__init__(*args, **kwargs) + self.signaller = Signaller() + self.signaller.signal.connect(slotfunc) + + def emit(self, record): + s = self.format(record) + self.signaller.signal.emit(s, record) + + # + # This example uses QThreads, which means that the threads at the Python level + # are named something like "Dummy-1". The function below gets the Qt name of the + # current thread. + # + def ctname(): + return QtCore.QThread.currentThread().objectName() + + + # + # Used to generate random levels for logging. + # + LEVELS = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, + logging.CRITICAL) + + # + # This worker class represents work that is done in a thread separate to the + # main thread. The way the thread is kicked off to do work is via a button press + # that connects to a slot in the worker. + # + # Because the default threadName value in the LogRecord isn't much use, we add + # a qThreadName which contains the QThread name as computed above, and pass that + # value in an "extra" dictionary which is used to update the LogRecord with the + # QThread name. + # + # This example worker just outputs messages sequentially, interspersed with + # random delays of the order of a few seconds. + # + class Worker(QtCore.QObject): + @Slot() + def start(self): + extra = {'qThreadName': ctname() } + logger.debug('Started work', extra=extra) + i = 1 + # Let the thread run until interrupted. This allows reasonably clean + # thread termination. + while not QtCore.QThread.currentThread().isInterruptionRequested(): + delay = 0.5 + random.random() * 2 + time.sleep(delay) + level = random.choice(LEVELS) + logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + i += 1 + + # + # Implement a simple UI for this cookbook example. This contains: + # + # * A read-only text edit window which holds formatted log messages + # * A button to start work and log stuff in a separate thread + # * A button to log something from the main thread + # * A button to clear the log window + # + class Window(QtWidgets.QWidget): + + COLORS = { + logging.DEBUG: 'black', + logging.INFO: 'blue', + logging.WARNING: 'orange', + logging.ERROR: 'red', + logging.CRITICAL: 'purple', + } + + def __init__(self, app): + super(Window, self).__init__() + self.app = app + self.textedit = te = QtWidgets.QPlainTextEdit(self) + # Set whatever the default monospace font is for the platform + f = QtGui.QFont('nosuchfont') + f.setStyleHint(f.Monospace) + te.setFont(f) + te.setReadOnly(True) + PB = QtWidgets.QPushButton + self.work_button = PB('Start background work', self) + self.log_button = PB('Log a message at a random level', self) + self.clear_button = PB('Clear log window', self) + self.handler = h = QtHandler(self.update_status) + # Remember to use qThreadName rather than threadName in the format string. + fs = '%(asctime)s %(qThreadName)-12s %(levelname)-8s %(message)s' + formatter = logging.Formatter(fs) + h.setFormatter(formatter) + logger.addHandler(h) + # Set up to terminate the QThread when we exit + app.aboutToQuit.connect(self.force_quit) + + # Lay out all the widgets + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(te) + layout.addWidget(self.work_button) + layout.addWidget(self.log_button) + layout.addWidget(self.clear_button) + self.setFixedSize(900, 400) + + # Connect the non-worker slots and signals + self.log_button.clicked.connect(self.manual_update) + self.clear_button.clicked.connect(self.clear_display) + + # Start a new worker thread and connect the slots for the worker + self.start_thread() + self.work_button.clicked.connect(self.worker.start) + # Once started, the button should be disabled + self.work_button.clicked.connect(lambda : self.work_button.setEnabled(False)) + + def start_thread(self): + self.worker = Worker() + self.worker_thread = QtCore.QThread() + self.worker.setObjectName('Worker') + self.worker_thread.setObjectName('WorkerThread') # for qThreadName + self.worker.moveToThread(self.worker_thread) + # This will start an event loop in the worker thread + self.worker_thread.start() + + def kill_thread(self): + # Just tell the worker to stop, then tell it to quit and wait for that + # to happen + self.worker_thread.requestInterruption() + if self.worker_thread.isRunning(): + self.worker_thread.quit() + self.worker_thread.wait() + else: + print('worker has already exited.') + + def force_quit(self): + # For use when the window is closed + if self.worker_thread.isRunning(): + self.kill_thread() + + # The functions below update the UI and run in the main thread because + # that's where the slots are set up + + @Slot(str, logging.LogRecord) + def update_status(self, status, record): + color = self.COLORS.get(record.levelno, 'black') + s = '
%s
' % (color, status) + self.textedit.appendHtml(s) + + @Slot() + def manual_update(self): + # This function uses the formatted message passed in, but also uses + # information from the record to format the message in an appropriate + # color according to its severity (level). + level = random.choice(LEVELS) + extra = {'qThreadName': ctname() } + logger.log(level, 'Manually logged!', extra=extra) + + @Slot() + def clear_display(self): + self.textedit.clear() + + + def main(): + QtCore.QThread.currentThread().setObjectName('MainThread') + logging.getLogger().setLevel(logging.DEBUG) + app = QtWidgets.QApplication(sys.argv) + example = Window(app) + example.show() + sys.exit(app.exec_()) + + if __name__=='__main__': + main() diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 7a68ca89199c77..b4dd9206c9d524 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -128,10 +128,18 @@ look at that next. Be sure to try the following in a newly-started Python interpreter, and don't just continue from the session described above:: import logging - logging.basicConfig(filename='example.log',level=logging.DEBUG) + logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.DEBUG) logging.debug('This message should go to the log file') logging.info('So should this') logging.warning('And this, too') + logging.error('And non-ASCII stuff, too, like Øresund and Malmö') + +.. versionchanged:: 3.9 + The *encoding* argument was added. In earlier Python versions, or if not + specified, the encoding used is the default value used by :func:`open`. While + not shown in the above example, an *errors* argument can also now be passed, + which determines how encoding errors are handled. For available values and + the default, see the documentation for :func:`open`. And now if we open the file and look at what we have, we should find the log messages: @@ -141,6 +149,7 @@ messages: DEBUG:root:This message should go to the log file INFO:root:So should this WARNING:root:And this, too + ERROR:root:And non-ASCII stuff, too, like Øresund and Malmö This example also shows how you can set the logging level which acts as the threshold for tracking. In this case, because we set the threshold to @@ -1069,21 +1078,22 @@ need more precise control over what logging information is collected. Here's a list of things you can do to avoid processing during logging which you don't need: -+-----------------------------------------------+----------------------------------------+ -| What you don't want to collect | How to avoid collecting it | -+===============================================+========================================+ -| Information about where calls were made from. | Set ``logging._srcfile`` to ``None``. | -| | This avoids calling | -| | :func:`sys._getframe`, which may help | -| | to speed up your code in environments | -| | like PyPy (which can't speed up code | -| | that uses :func:`sys._getframe`), if | -| | and when PyPy supports Python 3.x. | -+-----------------------------------------------+----------------------------------------+ -| Threading information. | Set ``logging.logThreads`` to ``0``. | -+-----------------------------------------------+----------------------------------------+ -| Process information. | Set ``logging.logProcesses`` to ``0``. | -+-----------------------------------------------+----------------------------------------+ ++-----------------------------------------------------+---------------------------------------------------+ +| What you don't want to collect | How to avoid collecting it | ++=====================================================+===================================================+ +| Information about where calls were made from. | Set ``logging._srcfile`` to ``None``. | +| | This avoids calling :func:`sys._getframe`, which | +| | may help to speed up your code in environments | +| | like PyPy (which can't speed up code that uses | +| | :func:`sys._getframe`). | ++-----------------------------------------------------+---------------------------------------------------+ +| Threading information. | Set ``logging.logThreads`` to ``False``. | ++-----------------------------------------------------+---------------------------------------------------+ +| Current process ID (:func:`os.getpid`) | Set ``logging.logProcesses`` to ``False``. | ++-----------------------------------------------------+---------------------------------------------------+ +| Current process name when using ``multiprocessing`` | Set ``logging.logMultiprocessing`` to ``False``. | +| to manage multiple processes. | | ++-----------------------------------------------------+---------------------------------------------------+ Also note that the core logging module only includes the basic handlers. If you don't import :mod:`logging.handlers` and :mod:`logging.config`, they won't diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 3be6bb380d663b..1543823c104c28 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -31,20 +31,26 @@ are: #. Only worry about supporting Python 2.7 #. Make sure you have good test coverage (coverage.py_ can help; - ``pip install coverage``) + ``python -m pip install coverage``) #. Learn the differences between Python 2 & 3 -#. Use Futurize_ (or Modernize_) to update your code (e.g. ``pip install future``) +#. Use Futurize_ (or Modernize_) to update your code (e.g. ``python -m pip install future``) #. Use Pylint_ to help make sure you don't regress on your Python 3 support - (``pip install pylint``) + (``python -m pip install pylint``) #. Use caniusepython3_ to find out which of your dependencies are blocking your - use of Python 3 (``pip install caniusepython3``) + use of Python 3 (``python -m pip install caniusepython3``) #. Once your dependencies are no longer blocking you, use continuous integration to make sure you stay compatible with Python 2 & 3 (tox_ can help test - against multiple versions of Python; ``pip install tox``) + against multiple versions of Python; ``python -m pip install tox``) #. Consider using optional static type checking to make sure your type usage works in both Python 2 & 3 (e.g. use mypy_ to check your typing under both - Python 2 & Python 3). + Python 2 & Python 3; ``python -m pip install mypy``). +.. note:: + + Note: Using ``python -m pip install`` guarantees that the ``pip`` you invoke + is the one installed for the Python currently in use, whether it be + a system-wide ``pip`` or one installed within a + :ref:`virtual environment `. Details ======= @@ -71,7 +77,7 @@ Drop support for Python 2.6 and older While you can make Python 2.5 work with Python 3, it is **much** easier if you only have to work with Python 2.7. If dropping Python 2.5 is not an option then the six_ project can help you support Python 2.5 & 3 simultaneously -(``pip install six``). Do realize, though, that nearly all the projects listed +(``python -m pip install six``). Do realize, though, that nearly all the projects listed in this HOWTO will not be available to you. If you are able to skip Python 2.5 and older, then the required changes @@ -119,7 +125,7 @@ Once you have your code well-tested you are ready to begin porting your code to Python 3! But to fully understand how your code is going to change and what you want to look out for while you code, you will want to learn what changes Python 3 makes in terms of Python 2. Typically the two best ways of doing that -is reading the `"What's New"`_ doc for each release of Python 3 and the +is reading the :ref:`"What's New" ` doc for each release of Python 3 and the `Porting to Python 3`_ book (which is free online). There is also a handy `cheat sheet`_ from the Python-Future project. @@ -302,10 +308,10 @@ If for some reason that doesn't work then you should make the version check be against Python 2 and not Python 3. To help explain this, let's look at an example. -Let's pretend that you need access to a feature of importlib_ that +Let's pretend that you need access to a feature of :mod:`importlib` that is available in Python's standard library since Python 3.3 and available for Python 2 through importlib2_ on PyPI. You might be tempted to write code to -access e.g. the ``importlib.abc`` module by doing the following:: +access e.g. the :mod:`importlib.abc` module by doing the following:: import sys @@ -426,12 +432,10 @@ can also explicitly state whether your APIs use textual or binary data, helping to make sure everything functions as expected in both versions of Python. -.. _2to3: https://docs.python.org/3/library/2to3.html .. _caniusepython3: https://pypi.org/project/caniusepython3 .. _cheat sheet: http://python-future.org/compatible_idioms.html .. _coverage.py: https://pypi.org/project/coverage .. _Futurize: http://python-future.org/automatic_conversion.html -.. _importlib: https://docs.python.org/3/library/importlib.html#module-importlib .. _importlib2: https://pypi.org/project/importlib2 .. _Modernize: https://python-modernize.readthedocs.io/ .. _mypy: http://mypy-lang.org/ @@ -447,6 +451,4 @@ to make sure everything functions as expected in both versions of Python. .. _tox: https://pypi.org/project/tox .. _trove classifier: https://pypi.org/classifiers -.. _"What's New": https://docs.python.org/3/whatsnew/index.html - .. _Why Python 3 exists: https://snarky.ca/why-python-3-exists diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index bc71d85a83e921..d6ed128e073fd6 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -317,9 +317,9 @@ know about the mechanics of using sockets. You'll still use the same calls, in much the same ways. It's just that, if you do it right, your app will be almost inside-out. -In Python, you use ``socket.setblocking(0)`` to make it non-blocking. In C, it's +In Python, you use ``socket.setblocking(False)`` to make it non-blocking. In C, it's more complex, (for one thing, you'll need to choose between the BSD flavor -``O_NONBLOCK`` and the almost indistinguishable Posix flavor ``O_NDELAY``, which +``O_NONBLOCK`` and the almost indistinguishable POSIX flavor ``O_NDELAY``, which is completely different from ``TCP_NODELAY``), but it's the exact same idea. You do this after creating the socket, but before using it. (Actually, if you're nuts, you can switch back and forth.) diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index 1d6d5c45b4d9f9..a8efe65353d6e1 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -43,16 +43,18 @@ Key Functions ============= Both :meth:`list.sort` and :func:`sorted` have a *key* parameter to specify a -function to be called on each list element prior to making comparisons. +function (or other callable) to be called on each list element prior to making +comparisons. For example, here's a case-insensitive string comparison: >>> sorted("This is a test string from Andrew".split(), key=str.lower) ['a', 'Andrew', 'from', 'is', 'string', 'test', 'This'] -The value of the *key* parameter should be a function that takes a single argument -and returns a key to use for sorting purposes. This technique is fast because -the key function is called exactly once for each input record. +The value of the *key* parameter should be a function (or other callable) that +takes a single argument and returns a key to use for sorting purposes. This +technique is fast because the key function is called exactly once for each +input record. A common pattern is to sort complex objects using some of the object's indices as keys. For example: diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 5339bf45bf0e80..e948c1e3c662d6 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -41,8 +41,9 @@ but these are two different characters that have different meanings. The Unicode standard describes how characters are represented by **code points**. A code point value is an integer in the range 0 to -0x10FFFF (about 1.1 million values, with some 110 thousand assigned so -far). In the standard and in this document, a code point is written +0x10FFFF (about 1.1 million values, the +`actual number assigned `_ +is less than that). In the standard and in this document, a code point is written using the notation ``U+265E`` to mean the character with value ``0x265e`` (9,822 in decimal). @@ -57,14 +58,14 @@ their corresponding code points: ... 007B '{'; LEFT CURLY BRACKET ... - 2167 'Ⅶ': ROMAN NUMERAL EIGHT - 2168 'Ⅸ': ROMAN NUMERAL NINE + 2167 'Ⅷ'; ROMAN NUMERAL EIGHT + 2168 'Ⅸ'; ROMAN NUMERAL NINE ... - 265E '♞': BLACK CHESS KNIGHT - 265F '♟': BLACK CHESS PAWN + 265E '♞'; BLACK CHESS KNIGHT + 265F '♟'; BLACK CHESS PAWN ... - 1F600 '😀': GRINNING FACE - 1F609 '😉': WINKING FACE + 1F600 '😀'; GRINNING FACE + 1F609 '😉'; WINKING FACE ... Strictly, these definitions imply that it's meaningless to say 'this is @@ -135,29 +136,34 @@ used than UTF-8.) UTF-8 uses the following rules: UTF-8 has several convenient properties: 1. It can handle any Unicode code point. -2. A Unicode string is turned into a sequence of bytes containing no embedded zero - bytes. This avoids byte-ordering issues, and means UTF-8 strings can be - processed by C functions such as ``strcpy()`` and sent through protocols that - can't handle zero bytes. +2. A Unicode string is turned into a sequence of bytes that contains embedded + zero bytes only where they represent the null character (U+0000). This means + that UTF-8 strings can be processed by C functions such as ``strcpy()`` and sent + through protocols that can't handle zero bytes for anything other than + end-of-string markers. 3. A string of ASCII text is also valid UTF-8 text. 4. UTF-8 is fairly compact; the majority of commonly used characters can be represented with one or two bytes. 5. If bytes are corrupted or lost, it's possible to determine the start of the next UTF-8-encoded code point and resynchronize. It's also unlikely that random 8-bit data will look like valid UTF-8. - +6. UTF-8 is a byte oriented encoding. The encoding specifies that each + character is represented by a specific sequence of one or more bytes. This + avoids the byte-ordering issues that can occur with integer and word oriented + encodings, like UTF-16 and UTF-32, where the sequence of bytes varies depending + on the hardware on which the string was encoded. References ---------- -The `Unicode Consortium site `_ has character charts, a +The `Unicode Consortium site `_ has character charts, a glossary, and PDF versions of the Unicode specification. Be prepared for some -difficult reading. `A chronology `_ of the +difficult reading. `A chronology `_ of the origin and development of Unicode is also available on the site. On the Computerphile Youtube channel, Tom Scott briefly -`discusses the history of Unicode and UTF-8 ` +`discusses the history of Unicode and UTF-8 `_ (9 minutes 36 seconds). To help understand the standard, Jukka Korpela has written `an introductory @@ -388,7 +394,7 @@ These are grouped into categories such as "Letter", "Number", "Punctuation", or from the above output, ``'Ll'`` means 'Letter, lowercase', ``'No'`` means "Number, other", ``'Mn'`` is "Mark, nonspacing", and ``'So'`` is "Symbol, other". See -`the General Category Values section of the Unicode Character Database documentation `_ for a +`the General Category Values section of the Unicode Character Database documentation `_ for a list of category codes. diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 046a88af62f0b3..38623371fbabff 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -601,5 +601,5 @@ This document was reviewed and revised by John Lee. scripts with a localhost server, I have to prevent urllib from using the proxy. .. [#] urllib opener for SSL proxy (CONNECT method): `ASPN Cookbook Recipe - `_. + `_. diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c index 13d16f5424ae4d..f361baf830dd1b 100644 --- a/Doc/includes/custom.c +++ b/Doc/includes/custom.c @@ -35,6 +35,11 @@ PyInit_custom(void) return NULL; Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c index 6477a19dafed75..5bacab7a2a9714 100644 --- a/Doc/includes/custom2.c +++ b/Doc/includes/custom2.c @@ -128,6 +128,11 @@ PyInit_custom2(void) return NULL; Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c index 213d0864ce1ca8..2b7a99ecf96c76 100644 --- a/Doc/includes/custom3.c +++ b/Doc/includes/custom3.c @@ -179,6 +179,11 @@ PyInit_custom3(void) return NULL; Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c index b0b2906dbdc863..584992fc5f1a8a 100644 --- a/Doc/includes/custom4.c +++ b/Doc/includes/custom4.c @@ -193,6 +193,11 @@ PyInit_custom4(void) return NULL; Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/includes/email-dir.py b/Doc/includes/email-dir.py index 0dcfbfb4025c85..2fc1570e654db6 100644 --- a/Doc/includes/email-dir.py +++ b/Doc/includes/email-dir.py @@ -41,7 +41,7 @@ def main(): directory = '.' # Create the message msg = EmailMessage() - msg['Subject'] = 'Contents of directory %s' % os.path.abspath(directory) + msg['Subject'] = f'Contents of directory {os.path.abspath(directory)}' msg['To'] = ', '.join(args.recipients) msg['From'] = args.sender msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' diff --git a/Doc/includes/email-mime.py b/Doc/includes/email-mime.py index c610242f11f843..6af2be0b08a48d 100644 --- a/Doc/includes/email-mime.py +++ b/Doc/includes/email-mime.py @@ -14,7 +14,7 @@ # family = the list of all recipients' email addresses msg['From'] = me msg['To'] = ', '.join(family) -msg.preamble = 'Our family reunion' +msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' # Open the files in binary mode. Use imghdr to figure out the # MIME subtype for each specific image. diff --git a/Doc/includes/email-simple.py b/Doc/includes/email-simple.py index f69ef40ff04c93..07dc30fd066eac 100644 --- a/Doc/includes/email-simple.py +++ b/Doc/includes/email-simple.py @@ -12,7 +12,7 @@ # me == the sender's email address # you == the recipient's email address -msg['Subject'] = 'The contents of %s' % textfile +msg['Subject'] = f'The contents of {textfile}' msg['From'] = me msg['To'] = you diff --git a/Doc/includes/email-unpack.py b/Doc/includes/email-unpack.py index e0a7f01f58bb59..c8cb0be4560830 100644 --- a/Doc/includes/email-unpack.py +++ b/Doc/includes/email-unpack.py @@ -43,7 +43,7 @@ def main(): if not ext: # Use a generic bag-of-bits extension ext = '.bin' - filename = 'part-%03d%s' % (counter, ext) + filename = f'part-{counter:03d}{ext}' counter += 1 with open(os.path.join(args.directory, filename), 'wb') as fp: fp.write(part.get_payload(decode=True)) diff --git a/Doc/includes/sqlite3/adapter_datetime.py b/Doc/includes/sqlite3/adapter_datetime.py index be33395100c325..d5221d80c35c8a 100644 --- a/Doc/includes/sqlite3/adapter_datetime.py +++ b/Doc/includes/sqlite3/adapter_datetime.py @@ -13,3 +13,5 @@ def adapt_datetime(ts): now = datetime.datetime.now() cur.execute("select ?", (now,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/adapter_point_1.py b/Doc/includes/sqlite3/adapter_point_1.py index 6b1af8415648a4..77daf8f16d227b 100644 --- a/Doc/includes/sqlite3/adapter_point_1.py +++ b/Doc/includes/sqlite3/adapter_point_1.py @@ -14,3 +14,5 @@ def __conform__(self, protocol): p = Point(4.0, -3.2) cur.execute("select ?", (p,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/adapter_point_2.py b/Doc/includes/sqlite3/adapter_point_2.py index d670700f0491b1..cb86331692b61d 100644 --- a/Doc/includes/sqlite3/adapter_point_2.py +++ b/Doc/includes/sqlite3/adapter_point_2.py @@ -15,3 +15,5 @@ def adapt_point(point): p = Point(4.0, -3.2) cur.execute("select ?", (p,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/connect_db_1.py b/Doc/includes/sqlite3/connect_db_1.py deleted file mode 100644 index 1b975232865aef..00000000000000 --- a/Doc/includes/sqlite3/connect_db_1.py +++ /dev/null @@ -1,3 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") diff --git a/Doc/includes/sqlite3/connect_db_2.py b/Doc/includes/sqlite3/connect_db_2.py deleted file mode 100644 index f9728b36135ed7..00000000000000 --- a/Doc/includes/sqlite3/connect_db_2.py +++ /dev/null @@ -1,3 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") diff --git a/Doc/includes/sqlite3/countcursors.py b/Doc/includes/sqlite3/countcursors.py index ef3e70a2a9cfc5..112f47703a2ff4 100644 --- a/Doc/includes/sqlite3/countcursors.py +++ b/Doc/includes/sqlite3/countcursors.py @@ -13,3 +13,5 @@ def cursor(self, *args, **kwargs): cur1 = con.cursor() cur2 = con.cursor() print(con.numcursors) + +con.close() diff --git a/Doc/includes/sqlite3/ctx_manager.py b/Doc/includes/sqlite3/ctx_manager.py index 7af4ad1ecfbcca..6db77d45046e1f 100644 --- a/Doc/includes/sqlite3/ctx_manager.py +++ b/Doc/includes/sqlite3/ctx_manager.py @@ -14,3 +14,7 @@ con.execute("insert into person(firstname) values (?)", ("Joe",)) except sqlite3.IntegrityError: print("couldn't add Joe twice") + +# Connection object used as context manager only commits or rollbacks transactions, +# so the connection object should be closed manually +con.close() diff --git a/Doc/includes/sqlite3/execsql_fetchonerow.py b/Doc/includes/sqlite3/execsql_fetchonerow.py index 078873bfc979a9..115bcb50c7c754 100644 --- a/Doc/includes/sqlite3/execsql_fetchonerow.py +++ b/Doc/includes/sqlite3/execsql_fetchonerow.py @@ -15,3 +15,5 @@ cur.execute(SELECT) for row in cur: print('%s is %d years old.' % (row[0], row[1])) + +con.close() diff --git a/Doc/includes/sqlite3/execsql_printall_1.py b/Doc/includes/sqlite3/execsql_printall_1.py index a4ce5c528149c5..19306e6e3ca7d1 100644 --- a/Doc/includes/sqlite3/execsql_printall_1.py +++ b/Doc/includes/sqlite3/execsql_printall_1.py @@ -11,3 +11,5 @@ # Retrieve all rows as a sequence and print that sequence: print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/execute_1.py b/Doc/includes/sqlite3/execute_1.py index f864a8984e4e90..3466b1265a5bf2 100644 --- a/Doc/includes/sqlite3/execute_1.py +++ b/Doc/includes/sqlite3/execute_1.py @@ -14,3 +14,5 @@ cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age}) print(cur.fetchone()) + +con.close() diff --git a/Doc/includes/sqlite3/execute_3.py b/Doc/includes/sqlite3/execute_3.py deleted file mode 100644 index 0353683fc70476..00000000000000 --- a/Doc/includes/sqlite3/execute_3.py +++ /dev/null @@ -1,12 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() - -who = "Yeltsin" -age = 72 - -cur.execute("select name_last, age from people where name_last=:who and age=:age", - locals()) -print(cur.fetchone()) diff --git a/Doc/includes/sqlite3/executemany_1.py b/Doc/includes/sqlite3/executemany_1.py index efae10637c7e8f..edf6f8b7ebe61a 100644 --- a/Doc/includes/sqlite3/executemany_1.py +++ b/Doc/includes/sqlite3/executemany_1.py @@ -22,3 +22,5 @@ def __next__(self): cur.execute("select c from characters") print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/executemany_2.py b/Doc/includes/sqlite3/executemany_2.py index 527358ebc28e1f..02a594c861e15b 100644 --- a/Doc/includes/sqlite3/executemany_2.py +++ b/Doc/includes/sqlite3/executemany_2.py @@ -13,3 +13,5 @@ def char_generator(): cur.execute("select c from characters") print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/executescript.py b/Doc/includes/sqlite3/executescript.py index 7e5358178d4c0a..aea8943fbee598 100644 --- a/Doc/includes/sqlite3/executescript.py +++ b/Doc/includes/sqlite3/executescript.py @@ -22,3 +22,4 @@ 1987 ); """) +con.close() diff --git a/Doc/includes/sqlite3/insert_more_people.py b/Doc/includes/sqlite3/insert_more_people.py index edbc79e7e5b6cc..10cf937243f6da 100644 --- a/Doc/includes/sqlite3/insert_more_people.py +++ b/Doc/includes/sqlite3/insert_more_people.py @@ -14,3 +14,5 @@ # The changes will not be saved unless the transaction is committed explicitly: con.commit() + +con.close() diff --git a/Doc/includes/sqlite3/load_extension.py b/Doc/includes/sqlite3/load_extension.py index b997c70668ace4..624cfe262f38b3 100644 --- a/Doc/includes/sqlite3/load_extension.py +++ b/Doc/includes/sqlite3/load_extension.py @@ -24,3 +24,5 @@ """) for row in con.execute("select rowid, name, ingredients from recipe where name match 'pie'"): print(row) + +con.close() diff --git a/Doc/includes/sqlite3/md5func.py b/Doc/includes/sqlite3/md5func.py index 0056b2d6ce84ef..16dc348bf001e2 100644 --- a/Doc/includes/sqlite3/md5func.py +++ b/Doc/includes/sqlite3/md5func.py @@ -9,3 +9,5 @@ def md5sum(t): cur = con.cursor() cur.execute("select md5(?)", (b"foo",)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/mysumaggr.py b/Doc/includes/sqlite3/mysumaggr.py index d2dfd2c0b98d3b..11f96395b6c485 100644 --- a/Doc/includes/sqlite3/mysumaggr.py +++ b/Doc/includes/sqlite3/mysumaggr.py @@ -18,3 +18,5 @@ def finalize(self): cur.execute("insert into test(i) values (2)") cur.execute("select mysum(i) from test") print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/parse_colnames.py b/Doc/includes/sqlite3/parse_colnames.py index cc68c76459ecea..5f01dbfe1cb524 100644 --- a/Doc/includes/sqlite3/parse_colnames.py +++ b/Doc/includes/sqlite3/parse_colnames.py @@ -6,3 +6,5 @@ cur.execute('select ? as "x [timestamp]"', (datetime.datetime.now(),)) dt = cur.fetchone()[0] print(dt, type(dt)) + +con.close() diff --git a/Doc/includes/sqlite3/pysqlite_datetime.py b/Doc/includes/sqlite3/pysqlite_datetime.py index 68d49358a578b3..5d843f906b3062 100644 --- a/Doc/includes/sqlite3/pysqlite_datetime.py +++ b/Doc/includes/sqlite3/pysqlite_datetime.py @@ -18,3 +18,5 @@ row = cur.fetchone() print("current_date", row[0], type(row[0])) print("current_timestamp", row[1], type(row[1])) + +con.close() diff --git a/Doc/includes/sqlite3/row_factory.py b/Doc/includes/sqlite3/row_factory.py index e436ffc6c80225..9de6e7b1b9052a 100644 --- a/Doc/includes/sqlite3/row_factory.py +++ b/Doc/includes/sqlite3/row_factory.py @@ -11,3 +11,5 @@ def dict_factory(cursor, row): cur = con.cursor() cur.execute("select 1 as a") print(cur.fetchone()["a"]) + +con.close() diff --git a/Doc/includes/sqlite3/rowclass.py b/Doc/includes/sqlite3/rowclass.py index 92b5ad60cb5791..fc60287069a854 100644 --- a/Doc/includes/sqlite3/rowclass.py +++ b/Doc/includes/sqlite3/rowclass.py @@ -10,3 +10,5 @@ assert row["name"] == row["nAmE"] assert row[1] == row["age"] assert row[1] == row["AgE"] + +con.close() diff --git a/Doc/includes/sqlite3/shortcut_methods.py b/Doc/includes/sqlite3/shortcut_methods.py index 71600d4f60c55e..98a39411495cba 100644 --- a/Doc/includes/sqlite3/shortcut_methods.py +++ b/Doc/includes/sqlite3/shortcut_methods.py @@ -18,3 +18,7 @@ print(row) print("I just deleted", con.execute("delete from person").rowcount, "rows") + +# close is not a shortcut method and it's not called automatically, +# so the connection object should be closed manually +con.close() diff --git a/Doc/includes/sqlite3/simple_tableprinter.py b/Doc/includes/sqlite3/simple_tableprinter.py index 231d8726cd436e..148a1707f948bc 100644 --- a/Doc/includes/sqlite3/simple_tableprinter.py +++ b/Doc/includes/sqlite3/simple_tableprinter.py @@ -24,3 +24,5 @@ print(fieldValue.ljust(FIELD_MAX_WIDTH), end=' ') print() # Finish the row with a newline. + +con.close() diff --git a/Doc/includes/sqlite3/text_factory.py b/Doc/includes/sqlite3/text_factory.py index 5f96cdb58da1ab..a857a155cdd4ff 100644 --- a/Doc/includes/sqlite3/text_factory.py +++ b/Doc/includes/sqlite3/text_factory.py @@ -25,3 +25,5 @@ cur.execute("select ?", ("bar",)) row = cur.fetchone() assert row[0] == "barfoo" + +con.close() diff --git a/Doc/includes/sublist.c b/Doc/includes/sublist.c index 76ff93948cfd67..b2c26e73ebaf7e 100644 --- a/Doc/includes/sublist.c +++ b/Doc/includes/sublist.c @@ -59,6 +59,11 @@ PyInit_sublist(void) return NULL; Py_INCREF(&SubListType); - PyModule_AddObject(m, "SubList", (PyObject *) &SubListType); + if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { + Py_DECREF(&SubListType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index 9f47899a198e01..9ada03cfc4a4cb 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -6,7 +6,7 @@ typedef struct _typeobject { /* Methods to implement standard operations */ destructor tp_dealloc; - printfunc tp_print; + Py_ssize_t tp_vectorcall_offset; getattrfunc tp_getattr; setattrfunc tp_setattr; PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) diff --git a/Doc/install/index.rst b/Doc/install/index.rst index f6a8cd6833a9ad..e6d5a3e6ebde60 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -1,4 +1,4 @@ -.. highlightlang:: none +.. highlight:: none .. _install-index: @@ -13,23 +13,10 @@ .. seealso:: :ref:`installing-index` - The up to date module installation documentations - -.. The audience for this document includes people who don't know anything - about Python and aren't about to learn the language just in order to - install and maintain it for their users, i.e. system administrators. - Thus, I have to be sure to explain the basics at some point: - sys.path and PYTHONPATH at least. Should probably give pointers to - other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc. - - Finally, it might be useful to include all the material from my "Care - and Feeding of a Python Installation" talk in here somewhere. Yow! - -This document describes the Python Distribution Utilities ("Distutils") from the -end-user's point-of-view, describing how to extend the capabilities of a -standard Python installation by building and installing third-party Python -modules and extensions. + The up to date module installation documentation. For regular Python + usage, you almost certainly want that document rather than this one. +.. include:: ../distutils/_setuptools_disclaimer.rst .. note:: @@ -46,59 +33,26 @@ modules and extensions. Introduction ============ -Although Python's extensive standard library covers many programming needs, -there often comes a time when you need to add some new functionality to your -Python installation in the form of third-party modules. This might be necessary -to support your own programming, or to support an application that you want to -use and that happens to be written in Python. - -In the past, there has been little support for adding third-party modules to an -existing Python installation. With the introduction of the Python Distribution -Utilities (Distutils for short) in Python 2.0, this changed. - -This document is aimed primarily at the people who need to install third-party -Python modules: end-users and system administrators who just need to get some -Python application running, and existing Python programmers who want to add some -new goodies to their toolbox. You don't need to know Python to read this -document; there will be some brief forays into using Python's interactive mode -to explore your installation, but that's it. If you're looking for information -on how to distribute your own Python modules so that others may use them, see -the :ref:`distutils-index` manual. :ref:`debug-setup-script` may also be of -interest. - - -.. _inst-trivial-install: - -Best case: trivial installation -------------------------------- - -In the best case, someone will have prepared a special version of the module -distribution you want to install that is targeted specifically at your platform -and is installed just like any other software on your platform. For example, -the module developer might make an executable installer available for Windows -users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE, -Mandrake, and many others), a Debian package for users of Debian-based Linux -systems, and so forth. - -In that case, you would download the installer appropriate to your platform and -do the obvious thing with it: run it if it's an executable installer, ``rpm ---install`` it if it's an RPM, etc. You don't need to run Python or a setup -script, you don't need to compile anything---you might not even need to read any -instructions (although it's always a good idea to do so anyway). - -Of course, things will not always be that easy. You might be interested in a -module distribution that doesn't have an easy-to-use installer for your -platform. In that case, you'll have to start with the source distribution -released by the module's author/maintainer. Installing from a source -distribution is not too hard, as long as the modules are packaged in the -standard way. The bulk of this document is about building and installing -modules from standard source distributions. +In Python 2.0, the ``distutils`` API was first added to the standard library. +This provided Linux distro maintainers with a standard way of converting +Python projects into Linux distro packages, and system administrators with a +standard way of installing them directly onto target systems. + +In the many years since Python 2.0 was released, tightly coupling the build +system and package installer to the language runtime release cycle has turned +out to be problematic, and it is now recommended that projects use the +``pip`` package installer and the ``setuptools`` build system, rather than +using ``distutils`` directly. + +See :ref:`installing-index` and :ref:`distributing-index` for more details. +This legacy documentation is being retained only until we're confident that the +``setuptools`` documentation covers everything needed. .. _inst-new-standard: -The new standard: Distutils ---------------------------- +Distutils based source distributions +------------------------------------ If you download a module source distribution, you can tell pretty quickly if it was packaged and distributed in the standard way, i.e. using the Distutils. @@ -578,7 +532,7 @@ scripts will wind up in :file:`/usr/local/python/bin`. If you want them in python setup.py install --install-scripts=/usr/local/bin -(This performs an installation using the "prefix scheme," where the prefix is +(This performs an installation using the "prefix scheme", where the prefix is whatever your Python interpreter was installed with--- :file:`/usr/local/python` in this case.) diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 747b9223b62cb8..31e9b0bde07244 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -1,4 +1,4 @@ -.. highlightlang:: none +.. highlight:: none .. _installing-index: @@ -53,7 +53,7 @@ Key terms evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation, and issue trackers on both `GitHub `__ and - `BitBucket `__. + `Bitbucket `__. * ``distutils`` is the original build and distribution system first added to the Python standard library in 1998. While direct use of ``distutils`` is being phased out, it still laid the foundation for the current packaging diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index fa4b0a9645531c..1d7bd262872905 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -9,9 +9,7 @@ of *fixers* to transform it into valid Python 3.x code. The standard library contains a rich set of fixers that will handle almost all code. 2to3 supporting library :mod:`lib2to3` is, however, a flexible and generic library, so it is -possible to write your own fixers for 2to3. :mod:`lib2to3` could also be -adapted to custom applications in which Python code needs to be edited -automatically. +possible to write your own fixers for 2to3. .. _2to3-using: @@ -102,7 +100,7 @@ presence of the ``from __future__ import print_function`` compiler directive, it modifies its internal grammar to interpret :func:`print` as a function. This change can also be enabled manually with the :option:`!-p` flag. Use :option:`!-p` to run fixers on code that already has had its print statements -converted. +converted. Also :option:`!-e` can be used to make :func:`exec` a function. The :option:`!-o` or :option:`!--output-dir` option allows specification of an alternate directory for processed output files to be written to. The @@ -456,7 +454,7 @@ and off individually. They are described here in more detail. ------------------------------- .. module:: lib2to3 - :synopsis: the 2to3 library + :synopsis: The 2to3 library .. moduleauthor:: Guido van Rossum .. moduleauthor:: Collin Winter @@ -466,9 +464,17 @@ and off individually. They are described here in more detail. -------------- +.. deprecated:: 3.10 + Python 3.9 will switch to a PEG parser (see :pep:`617`), and Python 3.10 may + include new language syntax that is not parsable by lib2to3's LL(1) parser. + The ``lib2to3`` module may be removed from the standard library in a future + Python version. Consider third-party alternatives such as `LibCST`_ or + `parso`_. + .. note:: The :mod:`lib2to3` API should be considered unstable and may change drastically in the future. -.. XXX What is the public interface anyway? +.. _LibCST: https://libcst.readthedocs.io/ +.. _parso: https://parso.readthedocs.io/ diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index e3d749e6017847..41399942d30308 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -90,7 +90,7 @@ language using this mechanism: | generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | | | | | *StopIteration handling inside generators* | +------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | 4.0 | :pep:`563`: | +| annotations | 3.7.0b1 | 3.10 | :pep:`563`: | | | | | *Postponed evaluation of annotations* | +------------------+-------------+--------------+---------------------------------------------+ diff --git a/Doc/library/_dummy_thread.rst b/Doc/library/_dummy_thread.rst deleted file mode 100644 index 7dccbc55475aae..00000000000000 --- a/Doc/library/_dummy_thread.rst +++ /dev/null @@ -1,22 +0,0 @@ -:mod:`_dummy_thread` --- Drop-in replacement for the :mod:`_thread` module -========================================================================== - -.. module:: _dummy_thread - :synopsis: Drop-in replacement for the _thread module. - -**Source code:** :source:`Lib/_dummy_thread.py` - -.. deprecated:: 3.7 - Python now always has threading enabled. Please use :mod:`_thread` - (or, better, :mod:`threading`) instead. - --------------- - -This module provides a duplicate interface to the :mod:`_thread` module. -It was meant to be imported when the :mod:`_thread` module was not provided -on a platform. - -Be careful to not use this module where deadlock might occur from a thread being -created that blocks waiting for another thread to be created. This often occurs -with blocking I/O. - diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index acffabf24bad5f..bd653ab32bb9c4 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -43,18 +43,32 @@ This module defines the following constants and functions: .. function:: start_new_thread(function, args[, kwargs]) - Start a new thread and return its identifier. The thread executes the function - *function* with the argument list *args* (which must be a tuple). The optional - *kwargs* argument specifies a dictionary of keyword arguments. When the function - returns, the thread silently exits. When the function terminates with an - unhandled exception, a stack trace is printed and then the thread exits (but - other threads continue to run). + Start a new thread and return its identifier. The thread executes the + function *function* with the argument list *args* (which must be a tuple). + The optional *kwargs* argument specifies a dictionary of keyword arguments. + + When the function returns, the thread silently exits. + + When the function terminates with an unhandled exception, + :func:`sys.unraisablehook` is called to handle the exception. The *object* + attribute of the hook argument is *function*. By default, a stack trace is + printed and then the thread exits (but other threads continue to run). + + When the function raises a :exc:`SystemExit` exception, it is silently + ignored. + + .. versionchanged:: 3.8 + :func:`sys.unraisablehook` is now used to handle unhandled exceptions. .. function:: interrupt_main() - Raise a :exc:`KeyboardInterrupt` exception in the main thread. A subthread can - use this function to interrupt the main thread. + Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main + thread. A thread can use this function to interrupt the main thread. + + If :data:`signal.SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. .. function:: exit() @@ -85,6 +99,18 @@ This module defines the following constants and functions: may be recycled when a thread exits and another thread is created. +.. function:: get_native_id() + + Return the native integral Thread ID of the current thread assigned by the kernel. + This is a non-negative integer. + Its value may be used to uniquely identify this particular thread system-wide + (until the thread terminates, after which the value may be recycled by the OS). + + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. + + .. versionadded:: 3.8 + + .. function:: stack_size([size]) Return the thread stack size used when creating new threads. The optional diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst index 7328907730fb10..2e917cf7321b85 100644 --- a/Doc/library/aifc.rst +++ b/Doc/library/aifc.rst @@ -208,6 +208,7 @@ number of frames must be filled in. .. method:: aifc.tell() + :noindex: Return the current write position in the output file. Useful in combination with :meth:`setmark`. @@ -232,6 +233,7 @@ number of frames must be filled in. .. method:: aifc.close() + :noindex: Close the AIFF file. The header of the file is updated to reflect the actual size of the audio data. After calling this method, the object can no longer be diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index cef197f3055581..7a7a4cf94979a1 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -142,7 +142,7 @@ ArgumentParser objects formatter_class=argparse.HelpFormatter, \ prefix_chars='-', fromfile_prefix_chars=None, \ argument_default=None, conflict_handler='error', \ - add_help=True, allow_abbrev=True) + add_help=True, allow_abbrev=True, exit_on_error=True) Create a new :class:`ArgumentParser` object. All parameters should be passed as keyword arguments. Each parameter has its own more detailed description @@ -179,9 +179,19 @@ ArgumentParser objects * allow_abbrev_ - Allows long options to be abbreviated if the abbreviation is unambiguous. (default: ``True``) + * exit_on_error_ - Determines whether or not ArgumentParser exits with + error info when an error occurs. (default: ``True``) + .. versionchanged:: 3.5 *allow_abbrev* parameter was added. + .. versionchanged:: 3.8 + In previous versions, *allow_abbrev* also disabled grouping of short + flags such as ``-vv`` to mean ``-v -v``. + + .. versionchanged:: 3.9 + *exit_on_error* parameter was added. + The following sections describe how each of these are used. @@ -439,7 +449,7 @@ default values to each of the argument help messages:: >>> parser.add_argument('--foo', type=int, default=42, help='FOO!') >>> parser.add_argument('bar', nargs='*', default=[1, 2, 3], help='BAR!') >>> parser.print_help() - usage: PROG [-h] [--foo FOO] [bar [bar ...]] + usage: PROG [-h] [--foo FOO] [bar ...] positional arguments: bar BAR! (default: [1, 2, 3]) @@ -643,6 +653,28 @@ the help options:: +h, ++help show this help message and exit +exit_on_error +^^^^^^^^^^^^^ + +Normally, when you pass an invalid argument list to the :meth:`~ArgumentParser.parse_args` +method of an :class:`ArgumentParser`, it will exit with error info. + +If the user would like catch errors manually, the feature can be enable by setting +``exit_on_error`` to ``False``:: + + >>> parser = argparse.ArgumentParser(exit_on_error=False) + >>> parser.add_argument('--integers', type=int) + _StoreAction(option_strings=['--integers'], dest='integers', nargs=None, const=None, default=None, type=, choices=None, help=None, metavar=None) + >>> try: + ... parser.parse_args('--integers a'.split()) + ... except argparse.ArgumentError: + ... print('Catching an argumentError') + ... + Catching an argumentError + +.. versionadded:: 3.9 + + The add_argument() method ------------------------- @@ -778,10 +810,12 @@ how the command-line arguments should be handled. The supplied actions are: example, this is useful for increasing verbosity levels:: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--verbose', '-v', action='count') + >>> parser.add_argument('--verbose', '-v', action='count', default=0) >>> parser.parse_args(['-vvv']) Namespace(verbose=3) + Note, the *default* will be ``None`` unless explicitly set to *0*. + * ``'help'`` - This prints a complete help message for all the options in the current parser and then exits. By default a help action is automatically added to the parser. See :class:`ArgumentParser` for details of how the @@ -797,10 +831,31 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(['--version']) PROG 2.0 +* ``'extend'`` - This stores a list, and extends each argument value to the + list. + Example usage:: + + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument("--foo", action="extend", nargs="+", type=str) + >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"]) + Namespace(foo=['f1', 'f2', 'f3', 'f4']) + + .. versionadded:: 3.8 + You may also specify an arbitrary action by passing an Action subclass or -other object that implements the same interface. The recommended way to do -this is to extend :class:`Action`, overriding the ``__call__`` method -and optionally the ``__init__`` method. +other object that implements the same interface. The ``BooleanOptionalAction`` +is available in ``argparse`` and adds support for boolean actions such as +``--foo`` and ``--no-foo``:: + + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) + >>> parser.parse_args(['--no-foo']) + Namespace(foo=False) + +The recommended way to create a custom action is to extend :class:`Action`, +overriding the ``__call__`` method and optionally the ``__init__`` and +``format_usage`` methods. An example of a custom action:: @@ -906,19 +961,6 @@ values are: usage: PROG [-h] foo [foo ...] PROG: error: the following arguments are required: foo -.. _`argparse.REMAINDER`: - -* ``argparse.REMAINDER``. All the remaining command-line arguments are gathered - into a list. This is commonly useful for command line utilities that dispatch - to other command line utilities:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo') - >>> parser.add_argument('command') - >>> parser.add_argument('args', nargs=argparse.REMAINDER) - >>> print(parser.parse_args('--foo B cmd --arg1 XX ZZ'.split())) - Namespace(args=['--arg1', 'XX', 'ZZ'], command='cmd', foo='B') - If the ``nargs`` keyword argument is not provided, the number of arguments consumed is determined by the action_. Generally this means a single command-line argument will be consumed and a single item (not a list) will be produced. @@ -1089,9 +1131,22 @@ container should match the type_ specified:: usage: doors.py [-h] {1,2,3} doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) -Any object that supports the ``in`` operator can be passed as the *choices* -value, so :class:`dict` objects, :class:`set` objects, custom containers, -etc. are all supported. +Any container can be passed as the *choices* value, so :class:`list` objects, +:class:`set` objects, and custom containers are all supported. +This includes :class:`enum.Enum`, which could be used to restrain +argument's choices; if we reuse previous rock/paper/scissors game example, +this could be as follows:: + + >>> from enum import Enum + >>> class GameMove(Enum): + ... ROCK = 'rock' + ... PAPER = 'paper' + ... SCISSORS = 'scissors' + ... + >>> parser = argparse.ArgumentParser(prog='game.py') + >>> parser.add_argument('move', type=GameMove, choices=GameMove) + >>> parser.parse_args(['rock']) + Namespace(move=) required @@ -1107,8 +1162,8 @@ keyword argument to :meth:`~ArgumentParser.add_argument`:: >>> parser.parse_args(['--foo', 'BAR']) Namespace(foo='BAR') >>> parser.parse_args([]) - usage: argparse.py [-h] [--foo FOO] - argparse.py: error: option --foo is required + usage: [-h] --foo FOO + : error: the following arguments are required: --foo As the example shows, if an option is marked as ``required``, :meth:`~ArgumentParser.parse_args` will report an error if that option is not @@ -1321,6 +1376,9 @@ Action instances should be callable, so subclasses must override the The ``__call__`` method may perform arbitrary actions, but will typically set attributes on the ``namespace`` based on ``dest`` and ``values``. +Action subclasses can define a ``format_usage`` method that takes no argument +and return a string which will be used when printing the usage of the program. +If such method is not provided, a sensible default will be used. The parse_args() method ----------------------- @@ -1583,7 +1641,7 @@ Sub-commands stored; by default ``None`` and no value is stored * required_ - Whether or not a subcommand must be provided, by default - ``False``. + ``False`` (added in 3.7) * help_ - help for sub-parser group in help output, by default ``None`` @@ -1739,6 +1797,9 @@ Sub-commands >>> parser.parse_args(['2', 'frobble']) Namespace(subparser_name='2', y='frobble') + .. versionchanged:: 3.7 + New *required* keyword argument. + FileType objects ^^^^^^^^^^^^^^^^ @@ -1989,7 +2050,14 @@ Exiting methods .. method:: ArgumentParser.exit(status=0, message=None) This method terminates the program, exiting with the specified *status* - and, if given, it prints a *message* before that. + and, if given, it prints a *message* before that. The user can override + this method to handle these steps differently:: + + class ErrorCatchingArgumentParser(argparse.ArgumentParser): + def exit(self, status=0, message=None): + if status: + raise Exception(f'Exiting because of an error: {message}') + exit(status) .. method:: ArgumentParser.error(message) diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 4ac7bb5391a7a4..78020738bf4f75 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -22,7 +22,7 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'B'`` | unsigned char | int | 1 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'u'`` | Py_UNICODE | Unicode character | 2 | \(1) | +| ``'u'`` | wchar_t | Unicode character | 2 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'h'`` | signed short | int | 2 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -36,9 +36,9 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'L'`` | unsigned long | int | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'q'`` | signed long long | int | 8 | \(2) | +| ``'q'`` | signed long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'Q'`` | unsigned long long | int | 8 | \(2) | +| ``'Q'`` | unsigned long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'f'`` | float | float | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -48,21 +48,15 @@ defined: Notes: (1) - The ``'u'`` type code corresponds to Python's obsolete unicode character - (:c:type:`Py_UNICODE` which is :c:type:`wchar_t`). Depending on the - platform, it can be 16 bits or 32 bits. + It can be 16 bits or 32 bits depending on the platform. - ``'u'`` will be removed together with the rest of the :c:type:`Py_UNICODE` - API. + .. versionchanged:: 3.9 + ``array('u')`` now uses ``wchar_t`` as C type instead of deprecated + ``Py_UNICODE``. This change doesn't affect to its behavior because + ``Py_UNICODE`` is alias of ``wchar_t`` since Python 3.3. .. deprecated-removed:: 3.3 4.0 -(2) - The ``'q'`` and ``'Q'`` type codes are available only if - the platform C compiler used to build Python supports C :c:type:`long long`, - or, on Windows, :c:type:`__int64`. - - .. versionadded:: 3.3 The actual representation of values is determined by the machine architecture (strictly speaking, by the C implementation). The actual size can be accessed @@ -83,6 +77,7 @@ The module defines the following type: to add initial items to the array. Otherwise, the iterable initializer is passed to the :meth:`extend` method. + .. audit-event:: array.__new__ typecode,initializer array.array .. data:: typecodes @@ -175,11 +170,6 @@ The following data items and methods are also supported: a.append(x)`` except that if there is a type error, the array is unchanged. -.. method:: array.fromstring() - - Deprecated alias for :meth:`frombytes`. - - .. method:: array.fromunicode(s) Extends this array with data from the given unicode string. The array must @@ -237,11 +227,6 @@ The following data items and methods are also supported: Convert the array to an ordinary list with the same items. -.. method:: array.tostring() - - Deprecated alias for :meth:`tobytes`. - - .. method:: array.tounicode() Convert the array to a unicode string. The array must be a type ``'u'`` array; diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 1884bea80e8047..755c60fba64115 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -7,6 +7,10 @@ .. sectionauthor:: Martin v. Löwis .. sectionauthor:: Georg Brandl +.. testsetup:: + + import ast + **Source code:** :source:`Lib/ast.py` -------------- @@ -23,6 +27,17 @@ classes all inherit from :class:`ast.AST`. An abstract syntax tree can be compiled into a Python code object using the built-in :func:`compile` function. +.. _abstract-grammar: + +Abstract Grammar +---------------- + +The abstract grammar is currently defined as follows: + +.. literalinclude:: ../../Parser/Python.asdl + :language: asdl + + Node classes ------------ @@ -101,23 +116,1392 @@ Node classes node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), lineno=0, col_offset=0) +.. versionchanged:: 3.8 + + Class :class:`ast.Constant` is now used for all constants. + +.. versionchanged:: 3.9 + + Simple indices are represented by their value, extended slices are + represented as tuples. + .. deprecated:: 3.8 - Class :class:`ast.Constant` is now used for all constants. Old classes - :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, + Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, - but they will be removed in future Python releases. + but they will be removed in future Python releases. In the meantime, + instantiating them will return an instance of a different class. +.. deprecated:: 3.9 -.. _abstract-grammar: + Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still + available, but they will be removed in future Python releases. + In the meantime, instantiating them will return an instance of + a different class. -Abstract Grammar ----------------- -The abstract grammar is currently defined as follows: +Literals +^^^^^^^^ -.. literalinclude:: ../../Parser/Python.asdl - :language: none +.. class:: Constant(value) + + A constant value. The ``value`` attribute of the ``Constant`` literal contains the + Python object it represents. The values represented can be simple types + such as a number, string or ``None``, but also immutable container types + (tuples and frozensets) if all of their elements are constant. + + .. doctest:: + + >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) + Expression( + body=Constant(value=123)) + + +.. class:: FormattedValue(value, conversion, format_spec) + + Node representing a single formatting field in an f-string. If the string + contains a single formatting field and nothing else the node can be + isolated otherwise it appears in :class:`JoinedStr`. + + * ``value`` is any expression node (such as a literal, a variable, or a + function call). + * ``conversion`` is an integer: + + * -1: no formatting + * 115: ``!s`` string formatting + * 114: ``!r`` repr formatting + * 97: ``!a`` ascii formatting + + * ``format_spec`` is a :class:`JoinedStr` node representing the formatting + of the value, or ``None`` if no format was specified. Both + ``conversion`` and ``format_spec`` can be set at the same time. + + +.. class:: JoinedStr(values) + + An f-string, comprising a series of :class:`FormattedValue` and :class:`Constant` + nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('f"sin({a}) is {sin(a):.3}"', mode='eval'), indent=4)) + Expression( + body=JoinedStr( + values=[ + Constant(value='sin('), + FormattedValue( + value=Name(id='a', ctx=Load()), + conversion=-1), + Constant(value=') is '), + FormattedValue( + value=Call( + func=Name(id='sin', ctx=Load()), + args=[ + Name(id='a', ctx=Load())], + keywords=[]), + conversion=-1, + format_spec=JoinedStr( + values=[ + Constant(value='.3')]))])) + + +.. class:: List(elts, ctx) + Tuple(elts, ctx) + + A list or tuple. ``elts`` holds a list of nodes representing the elements. + ``ctx`` is :class:`Store` if the container is an assignment target (i.e. + ``(x,y)=something``), and :class:`Load` otherwise. + + .. doctest:: + + >>> print(ast.dump(ast.parse('[1, 2, 3]', mode='eval'), indent=4)) + Expression( + body=List( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + ctx=Load())) + >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) + Expression( + body=Tuple( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + ctx=Load())) + + +.. class:: Set(elts) + + A set. ``elts`` holds a list of nodes representing the set's elements. + + .. doctest:: + + >>> print(ast.dump(ast.parse('{1, 2, 3}', mode='eval'), indent=4)) + Expression( + body=Set( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)])) + + +.. class:: Dict(keys, values) + + A dictionary. ``keys`` and ``values`` hold lists of nodes representing the + keys and the values respectively, in matching order (what would be returned + when calling :code:`dictionary.keys()` and :code:`dictionary.values()`). + + When doing dictionary unpacking using dictionary literals the expression to be + expanded goes in the ``values`` list, with a ``None`` at the corresponding + position in ``keys``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('{"a":1, **d}', mode='eval'), indent=4)) + Expression( + body=Dict( + keys=[ + Constant(value='a'), + None], + values=[ + Constant(value=1), + Name(id='d', ctx=Load())])) + + +Variables +^^^^^^^^^ + +.. class:: Name(id, ctx) + + A variable name. ``id`` holds the name as a string, and ``ctx`` is one of + the following types. + + +.. class:: Load() + Store() + Del() + + Variable references can be used to load the value of a variable, to assign + a new value to it, or to delete it. Variable references are given a context + to distinguish these cases. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a'), indent=4)) + Module( + body=[ + Expr( + value=Name(id='a', ctx=Load()))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a = 1'), indent=4)) + Module( + body=[ + Assign( + targets=[ + Name(id='a', ctx=Store())], + value=Constant(value=1))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('del a'), indent=4)) + Module( + body=[ + Delete( + targets=[ + Name(id='a', ctx=Del())])], + type_ignores=[]) + + +.. class:: Starred(value, ctx) + + A ``*var`` variable reference. ``value`` holds the variable, typically a + :class:`Name` node. This type must be used when building a :class:`Call` + node with ``*args``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a, *b = it'), indent=4)) + Module( + body=[ + Assign( + targets=[ + Tuple( + elts=[ + Name(id='a', ctx=Store()), + Starred( + value=Name(id='b', ctx=Store()), + ctx=Store())], + ctx=Store())], + value=Name(id='it', ctx=Load()))], + type_ignores=[]) + + +Expressions +^^^^^^^^^^^ + +.. class:: Expr(value) + + When an expression, such as a function call, appears as a statement by itself + with its return value not used or stored, it is wrapped in this container. + ``value`` holds one of the other nodes in this section, a :class:`Constant`, a + :class:`Name`, a :class:`Lambda`, a :class:`Yield` or :class:`YieldFrom` node. + + .. doctest:: + + >>> print(ast.dump(ast.parse('-a'), indent=4)) + Module( + body=[ + Expr( + value=UnaryOp( + op=USub(), + operand=Name(id='a', ctx=Load())))], + type_ignores=[]) + + +.. class:: UnaryOp(op, operand) + + A unary operation. ``op`` is the operator, and ``operand`` any expression + node. + + +.. class:: UAdd + USub + Not + Invert + + Unary operator tokens. :class:`Not` is the ``not`` keyword, :class:`Invert` + is the ``~`` operator. + + .. doctest:: + + >>> print(ast.dump(ast.parse('not x', mode='eval'), indent=4)) + Expression( + body=UnaryOp( + op=Not(), + operand=Name(id='x', ctx=Load()))) + + +.. class:: BinOp(left, op, right) + + A binary operation (like addition or division). ``op`` is the operator, and + ``left`` and ``right`` are any expression nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) + Expression( + body=BinOp( + left=Name(id='x', ctx=Load()), + op=Add(), + right=Name(id='y', ctx=Load()))) + + +.. class:: Add + Sub + Mult + Div + FloorDiv + Mod + Pow + LShift + RShift + BitOr + BitXor + BitAnd + MatMult + + Binary operator tokens. + + +.. class:: BoolOp(op, values) + + A boolean operation, 'or' or 'and'. ``op`` is :class:`Or` or :class:`And`. + ``values`` are the values involved. Consecutive operations with the same + operator, such as ``a or b or c``, are collapsed into one node with several + values. + + This doesn't include ``not``, which is a :class:`UnaryOp`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x or y', mode='eval'), indent=4)) + Expression( + body=BoolOp( + op=Or(), + values=[ + Name(id='x', ctx=Load()), + Name(id='y', ctx=Load())])) + + +.. class:: And + Or + + Boolean operator tokens. + + +.. class:: Compare(left, ops, comparators) + + A comparison of two or more values. ``left`` is the first value in the + comparison, ``ops`` the list of operators, and ``comparators`` the list + of values after the first element in the comparison. + + .. doctest:: + + >>> print(ast.dump(ast.parse('1 <= a < 10', mode='eval'), indent=4)) + Expression( + body=Compare( + left=Constant(value=1), + ops=[ + LtE(), + Lt()], + comparators=[ + Name(id='a', ctx=Load()), + Constant(value=10)])) + + +.. class:: Eq + NotEq + Lt + LtE + Gt + GtE + Is + IsNot + In + NotIn + + Comparison operator tokens. + + +.. class:: Call(func, args, keywords, starargs, kwargs) + + A function call. ``func`` is the function, which will often be a + :class:`Name` or :class:`Attribute` object. Of the arguments: + + * ``args`` holds a list of the arguments passed by position. + * ``keywords`` holds a list of :class:`keyword` objects representing + arguments passed by keyword. + + When creating a ``Call`` node, ``args`` and ``keywords`` are required, but + they can be empty lists. ``starargs`` and ``kwargs`` are optional. + + .. doctest:: + + >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) + Expression( + body=Call( + func=Name(id='func', ctx=Load()), + args=[ + Name(id='a', ctx=Load()), + Starred( + value=Name(id='d', ctx=Load()), + ctx=Load())], + keywords=[ + keyword( + arg='b', + value=Name(id='c', ctx=Load())), + keyword( + value=Name(id='e', ctx=Load()))])) + + +.. class:: keyword(arg, value) + + A keyword argument to a function call or class definition. ``arg`` is a raw + string of the parameter name, ``value`` is a node to pass in. + + +.. class:: IfExp(test, body, orelse) + + An expression such as ``a if b else c``. Each field holds a single node, so + in the following example, all three are :class:`Name` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) + Expression( + body=IfExp( + test=Name(id='b', ctx=Load()), + body=Name(id='a', ctx=Load()), + orelse=Name(id='c', ctx=Load()))) + + +.. class:: Attribute(value, attr, ctx) + + Attribute access, e.g. ``d.keys``. ``value`` is a node, typically a + :class:`Name`. ``attr`` is a bare string giving the name of the attribute, + and ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` according to how + the attribute is acted on. + + .. doctest:: + + >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) + Expression( + body=Attribute( + value=Name(id='snake', ctx=Load()), + attr='colour', + ctx=Load())) + + +.. class:: NamedExpr(target, value) + + A named expression. This AST node is produced by the assignment expressions + operator (also known as the walrus operator). As opposed to the :class:`Assign` + node in which the first argument can be multiple nodes, in this case both + ``target`` and ``value`` must be single nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('(x := 4)', mode='eval'), indent=4)) + Expression( + body=NamedExpr( + target=Name(id='x', ctx=Store()), + value=Constant(value=4))) + + +Subscripting +~~~~~~~~~~~~ + +.. class:: Subscript(value, slice, ctx) + + A subscript, such as ``l[1]``. ``value`` is the subscripted object + (usually sequence or mapping). ``slice`` is an index, slice or key. + It can be a :class:`Tuple` and contain a :class:`Slice`. + ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` + according to the action performed with the subscript. + + .. doctest:: + + >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) + Expression( + body=Subscript( + value=Name(id='l', ctx=Load()), + slice=Tuple( + elts=[ + Slice( + lower=Constant(value=1), + upper=Constant(value=2)), + Constant(value=3)], + ctx=Load()), + ctx=Load())) + + +.. class:: Slice(lower, upper, step) + + Regular slicing (on the form ``lower:upper`` or ``lower:upper:step``). + Can occur only inside the *slice* field of :class:`Subscript`, either + directly or as an element of :class:`Tuple`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) + Expression( + body=Subscript( + value=Name(id='l', ctx=Load()), + slice=Slice( + lower=Constant(value=1), + upper=Constant(value=2)), + ctx=Load())) + + +Comprehensions +~~~~~~~~~~~~~~ + +.. class:: ListComp(elt, generators) + SetComp(elt, generators) + GeneratorExp(elt, generators) + DictComp(key, value, generators) + + List and set comprehensions, generator expressions, and dictionary + comprehensions. ``elt`` (or ``key`` and ``value``) is a single node + representing the part that will be evaluated for each item. + + ``generators`` is a list of :class:`comprehension` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) + Expression( + body=ListComp( + elt=Name(id='x', ctx=Load()), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) + Expression( + body=DictComp( + key=Name(id='x', ctx=Load()), + value=BinOp( + left=Name(id='x', ctx=Load()), + op=Pow(), + right=Constant(value=2)), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) + Expression( + body=SetComp( + elt=Name(id='x', ctx=Load()), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + + +.. class:: comprehension(target, iter, ifs, is_async) + + One ``for`` clause in a comprehension. ``target`` is the reference to use for + each element - typically a :class:`Name` or :class:`Tuple` node. ``iter`` + is the object to iterate over. ``ifs`` is a list of test expressions: each + ``for`` clause can have multiple ``ifs``. + + ``is_async`` indicates a comprehension is asynchronous (using an + ``async for`` instead of ``for``). The value is an integer (0 or 1). + + .. doctest:: + + >>> print(ast.dump(ast.parse('[ord(c) for line in file for c in line]', mode='eval'), + ... indent=4)) # Multiple comprehensions in one. + Expression( + body=ListComp( + elt=Call( + func=Name(id='ord', ctx=Load()), + args=[ + Name(id='c', ctx=Load())], + keywords=[]), + generators=[ + comprehension( + target=Name(id='line', ctx=Store()), + iter=Name(id='file', ctx=Load()), + ifs=[], + is_async=0), + comprehension( + target=Name(id='c', ctx=Store()), + iter=Name(id='line', ctx=Load()), + ifs=[], + is_async=0)])) + + >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), + ... indent=4)) # generator comprehension + Expression( + body=GeneratorExp( + elt=BinOp( + left=Name(id='n', ctx=Load()), + op=Pow(), + right=Constant(value=2)), + generators=[ + comprehension( + target=Name(id='n', ctx=Store()), + iter=Name(id='it', ctx=Load()), + ifs=[ + Compare( + left=Name(id='n', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=5)]), + Compare( + left=Name(id='n', ctx=Load()), + ops=[ + Lt()], + comparators=[ + Constant(value=10)])], + is_async=0)])) + + >>> print(ast.dump(ast.parse('[i async for i in soc]', mode='eval'), + ... indent=4)) # Async comprehension + Expression( + body=ListComp( + elt=Name(id='i', ctx=Load()), + generators=[ + comprehension( + target=Name(id='i', ctx=Store()), + iter=Name(id='soc', ctx=Load()), + ifs=[], + is_async=1)])) + +Statements +^^^^^^^^^^ + +.. class:: Assign(targets, value, type_comment) + + An assignment. ``targets`` is a list of nodes, and ``value`` is a single node. + + Multiple nodes in ``targets`` represents assigning the same value to each. + Unpacking is represented by putting a :class:`Tuple` or :class:`List` + within ``targets``. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a = b = 1'), indent=4)) # Multiple assignment + Module( + body=[ + Assign( + targets=[ + Name(id='a', ctx=Store()), + Name(id='b', ctx=Store())], + value=Constant(value=1))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking + Module( + body=[ + Assign( + targets=[ + Tuple( + elts=[ + Name(id='a', ctx=Store()), + Name(id='b', ctx=Store())], + ctx=Store())], + value=Name(id='c', ctx=Load()))], + type_ignores=[]) + + +.. class:: AnnAssign(target, annotation, value, simple) + + An assignment with a type annotation. ``target`` is a single node and can + be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. + ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` + node. ``value`` is a single optional node. ``simple`` is a boolean integer + set to True for a :class:`Name` node in ``target`` that do not appear in + between parenthesis and are hence pure names and not expressions. + + .. doctest:: + + >>> print(ast.dump(ast.parse('c: int'), indent=4)) + Module( + body=[ + AnnAssign( + target=Name(id='c', ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=1)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis + Module( + body=[ + AnnAssign( + target=Name(id='a', ctx=Store()), + annotation=Name(id='int', ctx=Load()), + value=Constant(value=1), + simple=0)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation + Module( + body=[ + AnnAssign( + target=Attribute( + value=Name(id='a', ctx=Load()), + attr='b', + ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=0)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation + Module( + body=[ + AnnAssign( + target=Subscript( + value=Name(id='a', ctx=Load()), + slice=Constant(value=1), + ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=0)], + type_ignores=[]) + + +.. class:: AugAssign(target, op, value) + + Augmented assignment, such as ``a += 1``. In the following example, + ``target`` is a :class:`Name` node for ``x`` (with the :class:`Store` + context), ``op`` is :class:`Add`, and ``value`` is a :class:`Constant` with + value for 1. + + The ``target`` attribute connot be of class :class:`Tuple` or :class:`List`, + unlike the targets of :class:`Assign`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x += 2'), indent=4)) + Module( + body=[ + AugAssign( + target=Name(id='x', ctx=Store()), + op=Add(), + value=Constant(value=2))], + type_ignores=[]) + + +.. class:: Raise(exc, cause) + + A ``raise`` statement. ``exc`` is the exception object to be raised, normally a + :class:`Call` or :class:`Name`, or ``None`` for a standalone ``raise``. + ``cause`` is the optional part for ``y`` in ``raise x from y``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('raise x from y'), indent=4)) + Module( + body=[ + Raise( + exc=Name(id='x', ctx=Load()), + cause=Name(id='y', ctx=Load()))], + type_ignores=[]) + + +.. class:: Assert(test, msg) + + An assertion. ``test`` holds the condition, such as a :class:`Compare` node. + ``msg`` holds the failure message. + + .. doctest:: + + >>> print(ast.dump(ast.parse('assert x,y'), indent=4)) + Module( + body=[ + Assert( + test=Name(id='x', ctx=Load()), + msg=Name(id='y', ctx=Load()))], + type_ignores=[]) + + +.. class:: Delete(targets) + + Represents a ``del`` statement. ``targets`` is a list of nodes, such as + :class:`Name`, :class:`Attribute` or :class:`Subscript` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('del x,y,z'), indent=4)) + Module( + body=[ + Delete( + targets=[ + Name(id='x', ctx=Del()), + Name(id='y', ctx=Del()), + Name(id='z', ctx=Del())])], + type_ignores=[]) + + +.. class:: Pass() + + A ``pass`` statement. + + .. doctest:: + + >>> print(ast.dump(ast.parse('pass'), indent=4)) + Module( + body=[ + Pass()], + type_ignores=[]) + + +Other statements which are only applicable inside functions or loops are +described in other sections. + +Imports +~~~~~~~ + +.. class:: Import(names) + + An import statement. ``names`` is a list of :class:`alias` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('import x,y,z'), indent=4)) + Module( + body=[ + Import( + names=[ + alias(name='x'), + alias(name='y'), + alias(name='z')])], + type_ignores=[]) + + +.. class:: ImportFrom(module, names, level) + + Represents ``from x import y``. ``module`` is a raw string of the 'from' name, + without any leading dots, or ``None`` for statements such as ``from . import foo``. + ``level`` is an integer holding the level of the relative import (0 means + absolute import). + + .. doctest:: + + >>> print(ast.dump(ast.parse('from y import x,y,z'), indent=4)) + Module( + body=[ + ImportFrom( + module='y', + names=[ + alias(name='x'), + alias(name='y'), + alias(name='z')], + level=0)], + type_ignores=[]) + + +.. class:: alias(name, asname) + + Both parameters are raw strings of the names. ``asname`` can be ``None`` if + the regular name is to be used. + + .. doctest:: + + >>> print(ast.dump(ast.parse('from ..foo.bar import a as b, c'), indent=4)) + Module( + body=[ + ImportFrom( + module='foo.bar', + names=[ + alias(name='a', asname='b'), + alias(name='c')], + level=2)], + type_ignores=[]) + +Control flow +^^^^^^^^^^^^ + +.. note:: + Optional clauses such as ``else`` are stored as an empty list if they're + not present. + +.. class:: If(test, body, orelse) + + An ``if`` statement. ``test`` holds a single node, such as a :class:`Compare` + node. ``body`` and ``orelse`` each hold a list of nodes. + + ``elif`` clauses don't have a special representation in the AST, but rather + appear as extra :class:`If` nodes within the ``orelse`` section of the + previous one. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... if x: + ... ... + ... elif y: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + If( + test=Name(id='x', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + If( + test=Name(id='y', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + +.. class:: For(target, iter, body, orelse, type_comment) + + A ``for`` loop. ``target`` holds the variable(s) the loop assigns to, as a + single :class:`Name`, :class:`Tuple` or :class:`List` node. ``iter`` holds + the item to be looped over, again as a single node. ``body`` and ``orelse`` + contain lists of nodes to execute. Those in ``orelse`` are executed if the + loop finishes normally, rather than via a ``break`` statement. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... for x in y: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + For( + target=Name(id='x', ctx=Store()), + iter=Name(id='y', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: While(test, body, orelse) + + A ``while`` loop. ``test`` holds the condition, such as a :class:`Compare` + node. + + .. doctest:: + + >> print(ast.dump(ast.parse(""" + ... while x: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + While( + test=Name(id='x', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: Break + Continue + + The ``break`` and ``continue`` statements. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... for a in b: + ... if a > 5: + ... break + ... else: + ... continue + ... + ... """), indent=4)) + Module( + body=[ + For( + target=Name(id='a', ctx=Store()), + iter=Name(id='b', ctx=Load()), + body=[ + If( + test=Compare( + left=Name(id='a', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=5)]), + body=[ + Break()], + orelse=[ + Continue()])], + orelse=[])], + type_ignores=[]) + + +.. class:: Try(body, handlers, orelse, finalbody) + + ``try`` blocks. All attributes are list of nodes to execute, except for + ``handlers``, which is a list of :class:`ExceptHandler` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... try: + ... ... + ... except Exception: + ... ... + ... except OtherException as e: + ... ... + ... else: + ... ... + ... finally: + ... ... + ... """), indent=4)) + Module( + body=[ + Try( + body=[ + Expr( + value=Constant(value=Ellipsis))], + handlers=[ + ExceptHandler( + type=Name(id='Exception', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + ExceptHandler( + type=Name(id='OtherException', ctx=Load()), + name='e', + body=[ + Expr( + value=Constant(value=Ellipsis))])], + orelse=[ + Expr( + value=Constant(value=Ellipsis))], + finalbody=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: ExceptHandler(type, name, body) + + A single ``except`` clause. ``type`` is the exception type it will match, + typically a :class:`Name` node (or ``None`` for a catch-all ``except:`` clause). + ``name`` is a raw string for the name to hold the exception, or ``None`` if + the clause doesn't have ``as foo``. ``body`` is a list of nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... try: + ... a + 1 + ... except TypeError: + ... pass + ... """), indent=4)) + Module( + body=[ + Try( + body=[ + Expr( + value=BinOp( + left=Name(id='a', ctx=Load()), + op=Add(), + right=Constant(value=1)))], + handlers=[ + ExceptHandler( + type=Name(id='TypeError', ctx=Load()), + body=[ + Pass()])], + orelse=[], + finalbody=[])], + type_ignores=[]) + + +.. class:: With(items, body, type_comment) + + A ``with`` block. ``items`` is a list of :class:`withitem` nodes representing + the context managers, and ``body`` is the indented block inside the context. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + +.. class:: withitem(context_expr, optional_vars) + + A single context manager in a ``with`` block. ``context_expr`` is the context + manager, often a :class:`Call` node. ``optional_vars`` is a :class:`Name`, + :class:`Tuple` or :class:`List` for the ``as foo`` part, or ``None`` if that + isn't used. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... with a as b, c as d: + ... something(b, d) + ... """), indent=4)) + Module( + body=[ + With( + items=[ + withitem( + context_expr=Name(id='a', ctx=Load()), + optional_vars=Name(id='b', ctx=Store())), + withitem( + context_expr=Name(id='c', ctx=Load()), + optional_vars=Name(id='d', ctx=Store()))], + body=[ + Expr( + value=Call( + func=Name(id='something', ctx=Load()), + args=[ + Name(id='b', ctx=Load()), + Name(id='d', ctx=Load())], + keywords=[]))])], + type_ignores=[]) + + +Function and class definitions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment) + + A function definition. + + * ``name`` is a raw string of the function name. + * ``args`` is a :class:`arguments` node. + * ``body`` is the list of nodes inside the function. + * ``decorator_list`` is the list of decorators to be applied, stored outermost + first (i.e. the first in the list will be applied last). + * ``returns`` is the return annotation. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + +.. class:: Lambda(args, body) + + ``lambda`` is a minimal function definition that can be used inside an + expression. Unlike :class:`FunctionDef`, ``body`` holds a single node. + + .. doctest:: + + >>> print(ast.dump(ast.parse('lambda x,y: ...'), indent=4)) + Module( + body=[ + Expr( + value=Lambda( + args=arguments( + posonlyargs=[], + args=[ + arg(arg='x'), + arg(arg='y')], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=Constant(value=Ellipsis)))], + type_ignores=[]) + + +.. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) + + The arguments for a function. + + * ``posonlyargs``, ``args`` and ``kwonlyargs`` are lists of :class:`arg` nodes. + * ``vararg`` and ``kwarg`` are single :class:`arg` nodes, referring to the + ``*args, **kwargs`` parameters. + * ``kw_defaults`` is a list of default values for keyword-only arguments. If + one is ``None``, the corresponding argument is required. + * ``defaults`` is a list of default values for arguments that can be passed + positionally. If there are fewer defaults, they correspond to the last n + arguments. + + +.. class:: arg(arg, annotation, type_comment) + + A single argument in a list. ``arg`` is a raw string of the argument + name, ``annotation`` is its annotation, such as a :class:`Str` or + :class:`Name` node. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... @decorator1 + ... @decorator2 + ... def f(a: 'annotation', b=1, c=2, *d, e, f=3, **g) -> 'return annotation': + ... pass + ... """), indent=4)) + Module( + body=[ + FunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[ + arg( + arg='a', + annotation=Constant(value='annotation')), + arg(arg='b'), + arg(arg='c')], + vararg=arg(arg='d'), + kwonlyargs=[ + arg(arg='e'), + arg(arg='f')], + kw_defaults=[ + None, + Constant(value=3)], + kwarg=arg(arg='g'), + defaults=[ + Constant(value=1), + Constant(value=2)]), + body=[ + Pass()], + decorator_list=[ + Name(id='decorator1', ctx=Load()), + Name(id='decorator2', ctx=Load())], + returns=Constant(value='return annotation'))], + type_ignores=[]) + + +.. class:: Return(value) + + A ``return`` statement. + + .. doctest:: + + >>> print(ast.dump(ast.parse('return 4'), indent=4)) + Module( + body=[ + Return( + value=Constant(value=4))], + type_ignores=[]) + + +.. class:: Yield(value) + YieldFrom(value) + + A ``yield`` or ``yield from`` expression. Because these are expressions, they + must be wrapped in a :class:`Expr` node if the value sent back is not used. + + .. doctest:: + + >>> print(ast.dump(ast.parse('yield x'), indent=4)) + Module( + body=[ + Expr( + value=Yield( + value=Name(id='x', ctx=Load())))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('yield from x'), indent=4)) + Module( + body=[ + Expr( + value=YieldFrom( + value=Name(id='x', ctx=Load())))], + type_ignores=[]) + + +.. class:: Global(names) + Nonlocal(names) + + ``global`` and ``nonlocal`` statements. ``names`` is a list of raw strings. + + .. doctest:: + + >>> print(ast.dump(ast.parse('global x,y,z'), indent=4)) + Module( + body=[ + Global( + names=[ + 'x', + 'y', + 'z'])], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) + Module( + body=[ + Nonlocal( + names=[ + 'x', + 'y', + 'z'])], + type_ignores=[]) + + +.. class:: ClassDef(name, bases, keywords, starargs, kwargs, body, decorator_list) + + A class definition. + + * ``name`` is a raw string for the class name + * ``bases`` is a list of nodes for explicitly specified base classes. + * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'. + Other keywords will be passed to the metaclass, as per `PEP-3115 + `_. + * ``starargs`` and ``kwargs`` are each a single node, as in a function call. + starargs will be expanded to join the list of base classes, and kwargs will + be passed to the metaclass. + * ``body`` is a list of nodes representing the code within the class + definition. + * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... @decorator1 + ... @decorator2 + ... class Foo(base1, base2, metaclass=meta): + ... pass + ... """), indent=4)) + Module( + body=[ + ClassDef( + name='Foo', + bases=[ + Name(id='base1', ctx=Load()), + Name(id='base2', ctx=Load())], + keywords=[ + keyword( + arg='metaclass', + value=Name(id='meta', ctx=Load()))], + body=[ + Pass()], + decorator_list=[ + Name(id='decorator1', ctx=Load()), + Name(id='decorator2', ctx=Load())])], + type_ignores=[]) + +Async and await +^^^^^^^^^^^^^^^ + +.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment) + + An ``async def`` function definition. Has the same fields as + :class:`FunctionDef`. + + +.. class:: Await(value) + + An ``await`` expression. ``value`` is what it waits for. + Only valid in the body of an :class:`AsyncFunctionDef`. + +.. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... async def f(): + ... await other_func() + ... """), indent=4)) + Module( + body=[ + AsyncFunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=[ + Expr( + value=Await( + value=Call( + func=Name(id='other_func', ctx=Load()), + args=[], + keywords=[])))], + decorator_list=[])], + type_ignores=[]) + + +.. class:: AsyncFor(target, iter, body, orelse, type_comment) + AsyncWith(items, body, type_comment) + + ``async for`` loops and ``async with`` context managers. They have the same + fields as :class:`For` and :class:`With`, respectively. Only valid in the + body of an :class:`AsyncFunctionDef`. :mod:`ast` Helpers @@ -126,7 +1510,7 @@ The abstract grammar is currently defined as follows: Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. @@ -145,11 +1529,12 @@ and classes for traversing abstract syntax trees: modified to correspond to :pep:`484` "signature type comments", e.g. ``(str, int) -> List[str]``. - Also, setting ``feature_version`` to the minor version of an - earlier Python 3 version will attempt to parse using that version's - grammar. For example, setting ``feature_version=4`` will allow - the use of ``async`` and ``await`` as variable names. The lowest - supported value is 4; the highest is ``sys.version_info[1]``. + Also, setting ``feature_version`` to a tuple ``(major, minor)`` + will attempt to parse using that Python version's grammar. + Currently ``major`` must equal to ``3``. For example, setting + ``feature_version=(3, 4)`` will allow the use of ``async`` and + ``await`` as variable names. The lowest supported version is + ``(3, 4)``; the highest is ``sys.version_info[0:2]``. .. warning:: It is possible to crash the Python interpreter with a @@ -160,6 +1545,24 @@ and classes for traversing abstract syntax trees: Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. +.. function:: unparse(ast_obj) + + Unparse an :class:`ast.AST` object and generate a string with code + that would produce an equivalent :class:`ast.AST` object if parsed + back with :func:`ast.parse`. + + .. warning:: + The produced code string will not necessarily be equal to the original + code that generated the :class:`ast.AST` object (without any compiler + optimizations, such as constant tuples/frozensets). + + .. warning:: + Trying to unparse a highly complex expression would result with + :exc:`RecursionError`. + + .. versionadded:: 3.9 + + .. function:: literal_eval(node_or_string) Safely evaluate an expression node or a string containing a Python literal or @@ -180,6 +1583,9 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.2 Now allows bytes and set literals. + .. versionchanged:: 3.9 + Now supports creating empty sets with ``'set()'``. + .. function:: get_docstring(node, clean=True) @@ -274,6 +1680,13 @@ and classes for traversing abstract syntax trees: during traversal. For this a special visitor exists (:class:`NodeTransformer`) that allows modifications. + .. deprecated:: 3.8 + + Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, + :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated + now and will not be called in future Python versions. Add the + :meth:`visit_Constant` method to handle all constant nodes. + .. class:: NodeTransformer() @@ -292,11 +1705,11 @@ and classes for traversing abstract syntax trees: class RewriteName(NodeTransformer): def visit_Name(self, node): - return copy_location(Subscript( + return Subscript( value=Name(id='data', ctx=Load()), - slice=Index(value=Constant(value=node.id)), + slice=Constant(value=node.id), ctx=node.ctx - ), node) + ) Keep in mind that if the node you're operating on has child nodes you must either transform the child nodes yourself or call the :meth:`generic_visit` @@ -306,21 +1719,105 @@ and classes for traversing abstract syntax trees: statement nodes), the visitor may also return a list of nodes rather than just a single node. + If :class:`NodeTransformer` introduces new nodes (that weren't part of + original tree) without giving them location information (such as + :attr:`lineno`), :func:`fix_missing_locations` should be called with + the new sub-tree to recalculate the location information:: + + tree = ast.parse('foo', mode='eval') + new_tree = fix_missing_locations(RewriteName().visit(tree)) + Usually you use the transformer like this:: node = YourTransformer().visit(node) -.. function:: dump(node, annotate_fields=True, include_attributes=False) +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) Return a formatted dump of the tree in *node*. This is mainly useful for - debugging purposes. The returned string will show the names and the values - for fields. This makes the code impossible to evaluate, so if evaluation is - wanted *annotate_fields* must be set to ``False``. Attributes such as line + debugging purposes. If *annotate_fields* is true (by default), + the returned string will show the names and the values for fields. + If *annotate_fields* is false, the result string will be more compact by + omitting unambiguous field names. Attributes such as line numbers and column offsets are not dumped by default. If this is wanted, - *include_attributes* can be set to ``True``. + *include_attributes* can be set to true. + + If *indent* is a non-negative integer or string, then the tree will be + pretty-printed with that indent level. An indent level + of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) + selects the single line representation. Using a positive integer indent + indents that many spaces per level. If *indent* is a string (such as ``"\t"``), + that string is used to indent each level. + + .. versionchanged:: 3.9 + Added the *indent* option. + + +.. _ast-cli: + +Command-Line Usage +------------------ + +.. versionadded:: 3.9 + +The :mod:`ast` module can be executed as a script from the command line. +It is as simple as: + +.. code-block:: sh + + python -m ast [-m ] [-a] [infile] + +The following options are accepted: + +.. program:: ast + +.. cmdoption:: -h, --help + + Show the help message and exit. + +.. cmdoption:: -m + --mode + + Specify what kind of code must be compiled, like the *mode* argument + in :func:`parse`. + +.. cmdoption:: --no-type-comments + + Don't parse type comments. + +.. cmdoption:: -a, --include-attributes + + Include attributes such as line numbers and column offsets. + +.. cmdoption:: -i + --indent + + Indentation of nodes in AST (number of spaces). + +If :file:`infile` is specified its contents are parsed to AST and dumped +to stdout. Otherwise, the content is read from stdin. + .. seealso:: - `Green Tree Snakes `_, an external documentation resource, has good - details on working with Python ASTs. + `Green Tree Snakes `_, an external + documentation resource, has good details on working with Python ASTs. + + `ASTTokens `_ + annotates Python ASTs with the positions of tokens and text in the source + code that generated them. This is helpful for tools that make source code + transformations. + + `leoAst.py `_ unifies the + token-based and parse-tree-based views of python programs by inserting + two-way links between tokens and ast nodes. + + `LibCST `_ parses code as a Concrete Syntax + Tree that looks like an ast tree and keeps all formatting details. It's + useful for building automated refactoring (codemod) applications and + linters. + + `Parso `_ is a Python parser that supports + error recovery and round-trip parsing for different Python versions (in + multiple Python versions). Parso is also able to list multiple syntax errors + in your python file. \ No newline at end of file diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst index d5b5659abc65e2..047e5bbc58ccad 100644 --- a/Doc/library/asyncio-api-index.rst +++ b/Doc/library/asyncio-api-index.rst @@ -48,6 +48,9 @@ await on multiple things with timeouts. * - :class:`Task` - Task object. + * - :func:`to_thread` + - Asychronously run a function in a separate OS thread. + * - :func:`run_coroutine_threadsafe` - Schedule a coroutine from another OS thread. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index b7288036192979..02a00033152aba 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -25,7 +25,7 @@ There are several ways to enable asyncio debug mode: * Setting the :envvar:`PYTHONASYNCIODEBUG` environment variable to ``1``. -* Using the :option:`-X` ``dev`` Python command line option. +* Using the :ref:`Python Development Mode `. * Passing ``debug=True`` to :func:`asyncio.run`. @@ -73,7 +73,7 @@ event loop, no other Tasks can run in the same thread. When a Task executes an ``await`` expression, the running Task gets suspended, and the event loop executes the next Task. -To schedule a callback from a different OS thread, the +To schedule a :term:`callback` from another OS thread, the :meth:`loop.call_soon_threadsafe` method should be used. Example:: loop.call_soon_threadsafe(callback, *args) @@ -107,6 +107,16 @@ The :meth:`loop.run_in_executor` method can be used with a blocking code in a different OS thread without blocking the OS thread that the event loop runs in. +There is currently no way to schedule coroutines or callbacks directly +from a different process (such as one started with +:mod:`multiprocessing`). The :ref:`Event Loop Methods ` +section lists APIs that can read from pipes and watch file descriptors +without blocking the event loop. In addition, asyncio's +:ref:`Subprocess ` APIs provide a way to start a +process and communicate with it from the event loop. Lastly, the +aforementioned :meth:`loop.run_in_executor` method can also be used +with a :class:`concurrent.futures.ProcessPoolExecutor` to execute +code in a different process. .. _asyncio-handle-blocking: @@ -119,7 +129,7 @@ all concurrent asyncio Tasks and IO operations would be delayed by 1 second. An executor can be used to run a task in a different thread or even in -a different process to avoid blocking block the OS thread with the +a different process to avoid blocking the OS thread with the event loop. See the :meth:`loop.run_in_executor` method for more details. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index bf7c93a86fd041..b1e73189a7a4cf 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -5,6 +5,10 @@ Event Loop ========== +**Source code:** :source:`Lib/asyncio/events.py`, +:source:`Lib/asyncio/base_events.py` + +------------------------------------ .. rubric:: Preface @@ -34,8 +38,10 @@ an event loop: .. function:: get_event_loop() - Get the current event loop. If there is no current event loop set - in the current OS thread and :func:`set_event_loop` has not yet + Get the current event loop. + + If there is no current event loop set in the current OS thread, + the OS thread is main, and :func:`set_event_loop` has not yet been called, asyncio will create a new event loop and set it as the current one. @@ -167,14 +173,26 @@ Running and stopping the loop .. versionadded:: 3.6 +.. coroutinemethod:: loop.shutdown_default_executor() + + Schedule the closure of the default executor and wait for it to join all of + the threads in the :class:`ThreadPoolExecutor`. After calling this method, a + :exc:`RuntimeError` will be raised if :meth:`loop.run_in_executor` is called + while using the default executor. + + Note that there is no need to call this function when + :func:`asyncio.run` is used. + + .. versionadded:: 3.9 + Scheduling callbacks ^^^^^^^^^^^^^^^^^^^^ .. method:: loop.call_soon(callback, *args, context=None) - Schedule a *callback* to be called with *args* arguments at - the next iteration of the event loop. + Schedule the *callback* :term:`callback` to be called with + *args* arguments at the next iteration of the event loop. Callbacks are called in the order in which they are registered. Each callback will be called exactly once. @@ -341,7 +359,8 @@ Opening network connections host=None, port=None, \*, ssl=None, \ family=0, proto=0, flags=0, sock=None, \ local_addr=None, server_hostname=None, \ - ssl_handshake_timeout=None) + ssl_handshake_timeout=None, \ + happy_eyeballs_delay=None, interleave=None) Open a streaming transport connection to a given address specified by *host* and *port*. @@ -397,9 +416,27 @@ Opening network connections If given, these should all be integers from the corresponding :mod:`socket` module constants. + * *happy_eyeballs_delay*, if given, enables Happy Eyeballs for this + connection. It should + be a floating-point number representing the amount of time in seconds + to wait for a connection attempt to complete, before starting the next + attempt in parallel. This is the "Connection Attempt Delay" as defined + in :rfc:`8305`. A sensible default value recommended by the RFC is ``0.25`` + (250 milliseconds). + + * *interleave* controls address reordering when a host name resolves to + multiple IP addresses. + If ``0`` or unspecified, no reordering is done, and addresses are + tried in the order returned by :meth:`getaddrinfo`. If a positive integer + is specified, the addresses are interleaved by address family, and the + given integer is interpreted as "First Address Family Count" as defined + in :rfc:`8305`. The default is ``0`` if *happy_eyeballs_delay* is not + specified, and ``1`` if it is. + * *sock*, if given, should be an existing, already connected :class:`socket.socket` object to be used by the transport. - If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags* + If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags*, + *happy_eyeballs_delay*, *interleave* and *local_addr* should be specified. * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used @@ -410,6 +447,21 @@ Opening network connections to wait for the TLS handshake to complete before aborting the connection. ``60.0`` seconds if ``None`` (default). + .. versionadded:: 3.8 + + Added the *happy_eyeballs_delay* and *interleave* parameters. + + Happy Eyeballs Algorithm: Success with Dual-Stack Hosts. + When a server's IPv4 path and protocol are working, but the server's + IPv6 path and protocol are not working, a dual-stack client + application experiences significant connection delay compared to an + IPv4-only client. This is undesirable because it causes the dual- + stack client to have a worse user experience. This document + specifies requirements for algorithms that reduce this user-visible + delay and provides an algorithm. + + For more information: https://tools.ietf.org/html/rfc6555 + .. versionadded:: 3.7 The *ssl_handshake_timeout* parameter. @@ -435,6 +487,21 @@ Opening network connections reuse_address=None, reuse_port=None, \ allow_broadcast=None, sock=None) + .. note:: + The parameter *reuse_address* is no longer supported, as using + :py:data:`~sockets.SO_REUSEADDR` poses a significant security concern for + UDP. Explicitly passing ``reuse_address=True`` will raise an exception. + + When multiple processes with differing UIDs assign sockets to an + identical UDP socket address with ``SO_REUSEADDR``, incoming packets can + become randomly distributed among the sockets. + + For supported platforms, *reuse_port* can be used as a replacement for + similar functionality. With *reuse_port*, + :py:data:`~sockets.SO_REUSEPORT` is used instead, which specifically + prevents processes with differing UIDs from assigning sockets to the same + socket address. + Create a datagram connection. The socket family can be either :py:data:`~socket.AF_INET`, @@ -463,11 +530,6 @@ Opening network connections resolution. If given, these should all be integers from the corresponding :mod:`socket` module constants. - * *reuse_address* tells the kernel to reuse a local socket in - ``TIME_WAIT`` state, without waiting for its natural timeout to - expire. If not specified will automatically be set to ``True`` on - Unix. - * *reuse_port* tells the kernel to allow this endpoint to be bound to the same port as other existing endpoints are bound to, so long as they all set this flag when being created. This option is not supported on Windows @@ -482,8 +544,6 @@ Opening network connections transport. If specified, *local_addr* and *remote_addr* should be omitted (must be :const:`None`). - On Windows, with :class:`ProactorEventLoop`, this method is not supported. - See :ref:`UDP echo client protocol ` and :ref:`UDP echo server protocol ` examples. @@ -491,6 +551,13 @@ Opening network connections The *family*, *proto*, *flags*, *reuse_address*, *reuse_port, *allow_broadcast*, and *sock* parameters were added. + .. versionchanged:: 3.8.1 + The *reuse_address* parameter is no longer supported due to security + concerns. + + .. versionchanged:: 3.8 + Added support for Windows. + .. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ path=None, \*, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) @@ -1145,7 +1212,7 @@ Enabling debug mode .. versionchanged:: 3.7 - The new ``-X dev`` command line option can now also be used + The new :ref:`Python Development Mode ` can now also be used to enable the debug mode. .. seealso:: @@ -1195,32 +1262,52 @@ async/await code consider using the high-level Other parameters: - * *stdin*: either a file-like object representing a pipe to be - connected to the subprocess's standard input stream using - :meth:`~loop.connect_write_pipe`, or the - :const:`subprocess.PIPE` constant (default). By default a new - pipe will be created and connected. - - * *stdout*: either a file-like object representing the pipe to be - connected to the subprocess's standard output stream using - :meth:`~loop.connect_read_pipe`, or the - :const:`subprocess.PIPE` constant (default). By default a new pipe - will be created and connected. - - * *stderr*: either a file-like object representing the pipe to be - connected to the subprocess's standard error stream using - :meth:`~loop.connect_read_pipe`, or one of - :const:`subprocess.PIPE` (default) or :const:`subprocess.STDOUT` - constants. - - By default a new pipe will be created and connected. When - :const:`subprocess.STDOUT` is specified, the subprocess' standard - error stream will be connected to the same pipe as the standard - output stream. + * *stdin* can be any of these: + + * a file-like object representing a pipe to be connected to the + subprocess's standard input stream using + :meth:`~loop.connect_write_pipe` + * the :const:`subprocess.PIPE` constant (default) which will create a new + pipe and connect it, + * the value ``None`` which will make the subprocess inherit the file + descriptor from this process + * the :const:`subprocess.DEVNULL` constant which indicates that the + special :data:`os.devnull` file will be used + + * *stdout* can be any of these: + + * a file-like object representing a pipe to be connected to the + subprocess's standard output stream using + :meth:`~loop.connect_write_pipe` + * the :const:`subprocess.PIPE` constant (default) which will create a new + pipe and connect it, + * the value ``None`` which will make the subprocess inherit the file + descriptor from this process + * the :const:`subprocess.DEVNULL` constant which indicates that the + special :data:`os.devnull` file will be used + + * *stderr* can be any of these: + + * a file-like object representing a pipe to be connected to the + subprocess's standard error stream using + :meth:`~loop.connect_write_pipe` + * the :const:`subprocess.PIPE` constant (default) which will create a new + pipe and connect it, + * the value ``None`` which will make the subprocess inherit the file + descriptor from this process + * the :const:`subprocess.DEVNULL` constant which indicates that the + special :data:`os.devnull` file will be used + * the :const:`subprocess.STDOUT` constant which will connect the standard + error stream to the process' standard output stream * All other keyword arguments are passed to :class:`subprocess.Popen` - without interpretation, except for *bufsize*, *universal_newlines* - and *shell*, which should not be specified at all. + without interpretation, except for *bufsize*, *universal_newlines*, + *shell*, *text*, *encoding* and *errors*, which should not be specified + at all. + + The ``asyncio`` subprocess API does not support decoding the streams + as text. :func:`bytes.decode` can be used to convert the bytes returned + from the stream to text. See the constructor of the :class:`subprocess.Popen` class for documentation on other arguments. @@ -1411,7 +1498,7 @@ asyncio ships with two different event loop implementations: :class:`SelectorEventLoop` and :class:`ProactorEventLoop`. By default asyncio is configured to use :class:`SelectorEventLoop` -on all platforms. +on Unix and :class:`ProactorEventLoop` on Windows. .. class:: SelectorEventLoop @@ -1601,7 +1688,7 @@ using the :meth:`loop.add_signal_handler` method:: import os import signal - def ask_exit(signame): + def ask_exit(signame, loop): print("got signal %s: exit" % signame) loop.stop() @@ -1611,7 +1698,7 @@ using the :meth:`loop.add_signal_handler` method:: for signame in {'SIGINT', 'SIGTERM'}: loop.add_signal_handler( getattr(signal, signame), - functools.partial(ask_exit, signame)) + functools.partial(ask_exit, signame, loop)) await asyncio.sleep(3600) diff --git a/Doc/library/asyncio-exceptions.rst b/Doc/library/asyncio-exceptions.rst index e49577a203e8fb..7166d5c4bd88f9 100644 --- a/Doc/library/asyncio-exceptions.rst +++ b/Doc/library/asyncio-exceptions.rst @@ -7,6 +7,9 @@ Exceptions ========== +**Source code:** :source:`Lib/asyncio/exceptions.py` + +---------------------------------------------------- .. exception:: TimeoutError @@ -25,26 +28,9 @@ Exceptions when asyncio Tasks are cancelled. In almost all situations the exception must be re-raised. - .. important:: - - This exception is a subclass of :exc:`Exception`, so it can be - accidentally suppressed by an overly broad ``try..except`` block:: - - try: - await operation - except Exception: - # The cancellation is broken because the *except* block - # suppresses the CancelledError exception. - log.log('an error has occurred') - - Instead, the following pattern should be used:: + .. versionchanged:: 3.8 - try: - await operation - except asyncio.CancelledError: - raise - except Exception: - log.log('an error has occurred') + :exc:`CancelledError` is now a subclass of :class:`BaseException`. .. exception:: InvalidStateError diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 6e6e0137c1bdda..e1ac18eaf09882 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -7,6 +7,11 @@ Futures ======= +**Source code:** :source:`Lib/asyncio/futures.py`, +:source:`Lib/asyncio/base_futures.py` + +------------------------------------- + *Future* objects are used to bridge **low-level callback-based code** with high-level async/await code. @@ -35,7 +40,9 @@ Future Functions is used for the test.) * a :class:`Task` object wrapping *obj*, if *obj* is a - coroutine (:func:`iscoroutine` is used for the test.) + coroutine (:func:`iscoroutine` is used for the test); + in this case the coroutine will be scheduled by + ``ensure_future()``. * a :class:`Task` object that would await on *obj*, if *obj* is an awaitable (:func:`inspect.isawaitable` is used for the test.) @@ -163,7 +170,7 @@ Future Object Returns the number of callbacks removed, which is typically 1, unless a callback was added more than once. - .. method:: cancel() + .. method:: cancel(msg=None) Cancel the Future and schedule callbacks. @@ -171,6 +178,9 @@ Future Object Otherwise, change the Future's state to *cancelled*, schedule the callbacks, and return ``True``. + .. versionchanged:: 3.9 + Added the ``msg`` parameter. + .. method:: exception() Return the exception that was set on this Future. @@ -248,3 +258,6 @@ the Future has a result:: - asyncio Future is not compatible with the :func:`concurrent.futures.wait` and :func:`concurrent.futures.as_completed` functions. + + - :meth:`asyncio.Future.cancel` accepts an optional ``msg`` argument, + but :func:`concurrent.futures.cancel` does not. diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index 81d840e23277ea..390ee1969d096f 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -23,6 +23,12 @@ All Platforms Windows ======= +**Source code:** :source:`Lib/asyncio/proactor_events.py`, +:source:`Lib/asyncio/windows_events.py`, +:source:`Lib/asyncio/windows_utils.py` + +-------------------------------------- + .. versionchanged:: 3.8 On Windows, :class:`ProactorEventLoop` is now the default event loop. @@ -53,9 +59,6 @@ All event loops on Windows do not support the following methods: :class:`ProactorEventLoop` has the following limitations: -* The :meth:`loop.create_datagram_endpoint` method - is not supported. - * The :meth:`loop.add_reader` and :meth:`loop.add_writer` methods are not supported. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 6212df85dbc10a..d9d3232d2408b3 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -117,6 +117,7 @@ asyncio ships with the following built-in policies: .. availability:: Windows. +.. _asyncio-watchers: Process Watchers ================ @@ -129,10 +130,11 @@ In asyncio, child processes are created with :func:`create_subprocess_exec` and :meth:`loop.subprocess_exec` functions. -asyncio defines the :class:`AbstractChildWatcher` abstract base class, -which child watchers should implement, and has two different -implementations: :class:`SafeChildWatcher` (configured to be used -by default) and :class:`FastChildWatcher`. +asyncio defines the :class:`AbstractChildWatcher` abstract base class, which child +watchers should implement, and has four different implementations: +:class:`ThreadedChildWatcher` (configured to be used by default), +:class:`MultiLoopChildWatcher`, :class:`SafeChildWatcher`, and +:class:`FastChildWatcher`. See also the :ref:`Subprocess and Threads ` section. @@ -184,6 +186,15 @@ implementation used by the asyncio event loop: Note: loop may be ``None``. + .. method:: is_active() + + Return ``True`` if the watcher is ready to use. + + Spawning a subprocess with *inactive* current child watcher raises + :exc:`RuntimeError`. + + .. versionadded:: 3.8 + .. method:: close() Close the watcher. @@ -191,16 +202,48 @@ implementation used by the asyncio event loop: This method has to be called to ensure that underlying resources are cleaned-up. -.. class:: SafeChildWatcher +.. class:: ThreadedChildWatcher + + This implementation starts a new waiting thread for every subprocess spawn. + + It works reliably even when the asyncio event loop is run in a non-main OS thread. + + There is no noticeable overhead when handling a big number of children (*O(1)* each + time a child terminates), but stating a thread per process requires extra memory. + + This watcher is used by default. + + .. versionadded:: 3.8 + +.. class:: MultiLoopChildWatcher + + This implementation registers a :py:data:`SIGCHLD` signal handler on + instantiation. That can break third-party code that installs a custom handler for + `SIGCHLD`. signal). - This implementation avoids disrupting other code spawning processes + The watcher avoids disrupting other code spawning processes by polling every process explicitly on a :py:data:`SIGCHLD` signal. - This is a safe solution but it has a significant overhead when + There is no limitation for running subprocesses from different threads once the + watcher is installed. + + The solution is safe but it has a significant overhead when handling a big number of processes (*O(n)* each time a :py:data:`SIGCHLD` is received). - asyncio uses this safe implementation by default. + .. versionadded:: 3.8 + +.. class:: SafeChildWatcher + + This implementation uses active event loop from the main thread to handle + :py:data:`SIGCHLD` signal. If the main thread has no running event loop another + thread cannot spawn a subprocess (:exc:`RuntimeError` is raised). + + The watcher avoids disrupting other code spawning processes + by polling every process explicitly on a :py:data:`SIGCHLD` signal. + + This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O(N)* + complexity but requires a running event loop in the main thread to work. .. class:: FastChildWatcher @@ -211,6 +254,21 @@ implementation used by the asyncio event loop: There is no noticeable overhead when handling a big number of children (*O(1)* each time a child terminates). + This solution requires a running event loop in the main thread to work, as + :class:`SafeChildWatcher`. + +.. class:: PidfdChildWatcher + + This implementation polls process file descriptors (pidfds) to await child + process termination. In some respects, :class:`PidfdChildWatcher` is a + "Goldilocks" child watcher implementation. It doesn't require signals or + threads, doesn't interfere with any processes launched outside the event + loop, and scales linearly with the number of subprocesses launched by the + event loop. The main disadvantage is that pidfds are specific to Linux, and + only work on recent (5.3+) kernels. + + .. versionadded:: 3.9 + Custom Policies =============== diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index f08738dd62bb5c..9dbd3ab46a3f68 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -69,6 +69,10 @@ This documentation page contains the following sections: Transports ========== +**Source code:** :source:`Lib/asyncio/transports.py` + +---------------------------------------------------- + Transports are classes provided by :mod:`asyncio` in order to abstract various kinds of communication channels. @@ -431,6 +435,10 @@ Subprocess Transports Protocols ========= +**Source code:** :source:`Lib/asyncio/protocols.py` + +--------------------------------------------------- + asyncio provides a set of abstract base classes that should be used to implement network protocols. Those classes are meant to be used together with :ref:`transports `. @@ -580,9 +588,6 @@ Buffered Streaming Protocols ---------------------------- .. versionadded:: 3.7 - **Important:** this has been added to asyncio in Python 3.7 - *on a provisional basis*! This is as an experimental API that - might be changed or removed completely in Python 3.8. Buffered Protocols can be used with any event loop method that supports `Streaming Protocols`_. @@ -767,9 +772,8 @@ data, and waits until the connection is closed:: class EchoClientProtocol(asyncio.Protocol): - def __init__(self, message, on_con_lost, loop): + def __init__(self, message, on_con_lost): self.message = message - self.loop = loop self.on_con_lost = on_con_lost def connection_made(self, transport): @@ -793,7 +797,7 @@ data, and waits until the connection is closed:: message = 'Hello World!' transport, protocol = await loop.create_connection( - lambda: EchoClientProtocol(message, on_con_lost, loop), + lambda: EchoClientProtocol(message, on_con_lost), '127.0.0.1', 8888) # Wait until the protocol signals that the connection @@ -869,11 +873,10 @@ method, sends data and closes the transport when it receives the answer:: class EchoClientProtocol: - def __init__(self, message, loop): + def __init__(self, message, on_con_lost): self.message = message - self.loop = loop + self.on_con_lost = on_con_lost self.transport = None - self.on_con_lost = loop.create_future() def connection_made(self, transport): self.transport = transport @@ -899,13 +902,15 @@ method, sends data and closes the transport when it receives the answer:: # low-level APIs. loop = asyncio.get_running_loop() + on_con_lost = loop.create_future() message = "Hello World!" + transport, protocol = await loop.create_datagram_endpoint( - lambda: EchoClientProtocol(message, loop), + lambda: EchoClientProtocol(message, on_con_lost), remote_addr=('127.0.0.1', 9999)) try: - await protocol.on_con_lost + await on_con_lost finally: transport.close() @@ -927,9 +932,9 @@ Wait until a socket receives data using the class MyProtocol(asyncio.Protocol): - def __init__(self, loop): + def __init__(self, on_con_lost): self.transport = None - self.on_con_lost = loop.create_future() + self.on_con_lost = on_con_lost def connection_made(self, transport): self.transport = transport @@ -950,13 +955,14 @@ Wait until a socket receives data using the # Get a reference to the event loop as we plan to use # low-level APIs. loop = asyncio.get_running_loop() + on_con_lost = loop.create_future() # Create a pair of connected sockets rsock, wsock = socket.socketpair() # Register the socket to wait for data. transport, protocol = await loop.create_connection( - lambda: MyProtocol(loop), sock=rsock) + lambda: MyProtocol(on_con_lost), sock=rsock) # Simulate the reception of data from the network. loop.call_soon(wsock.send, 'abc'.encode()) @@ -987,7 +993,7 @@ loop.subprocess_exec() and SubprocessProtocol An example of a subprocess protocol used to get the output of a subprocess and to wait for the subprocess exit. -The subprocess is created by th :meth:`loop.subprocess_exec` method:: +The subprocess is created by the :meth:`loop.subprocess_exec` method:: import asyncio import sys diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index 7be1023c80cc66..524560b691d720 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -6,6 +6,10 @@ Queues ====== +**Source code:** :source:`Lib/asyncio/queues.py` + +------------------------------------------------ + asyncio queues are designed to be similar to classes of the :mod:`queue` module. Although asyncio queues are not thread-safe, they are designed to be used specifically in async/await code. @@ -32,6 +36,10 @@ Queue the queue is always known and can be returned by calling the :meth:`qsize` method. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + + This class is :ref:`not thread safe `. .. attribute:: maxsize diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index e686a6a1c4cd32..b76ed379c7f4c8 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -6,6 +6,10 @@ Streams ======= +**Source code:** :source:`Lib/asyncio/streams.py` + +------------------------------------------------- + Streams are high-level async/await-ready primitives to work with network connections. Streams allow sending and receiving data without using callbacks or low-level protocols and transports. @@ -22,13 +26,15 @@ streams:: '127.0.0.1', 8888) print(f'Send: {message!r}') - await writer.awrite(message.encode()) + writer.write(message.encode()) + await writer.drain() data = await reader.read(100) print(f'Received: {data.decode()!r}') print('Close the connection') - await writer.aclose() + writer.close() + await writer.wait_closed() asyncio.run(tcp_echo_client('Hello World!')) @@ -147,9 +153,6 @@ and work with streams: The *path* parameter can now be a :term:`path-like object`. ---------- - - StreamReader ============ @@ -226,28 +229,42 @@ StreamWriter directly; use :func:`open_connection` and :func:`start_server` instead. - .. coroutinemethod:: awrite(data) + .. method:: write(data) + + The method attempts to write the *data* to the underlying socket immediately. + If that fails, the data is queued in an internal write buffer until it can be + sent. + + The method should be used along with the ``drain()`` method:: - Write *data* to the stream. + stream.write(data) + await stream.drain() - The method respects flow control, execution is paused if the write - buffer reaches the high watermark. + .. method:: writelines(data) + + The method writes a list (or any iterable) of bytes to the underlying socket + immediately. + If that fails, the data is queued in an internal write buffer until it can be + sent. - .. versionadded:: 3.8 + The method should be used along with the ``drain()`` method:: - .. coroutinemethod:: aclose() + stream.writelines(lines) + await stream.drain() - Close the stream. + .. method:: close() - Wait until all closing actions are complete, e.g. SSL shutdown for - secure sockets. + The method closes the stream and the underlying socket. - .. versionadded:: 3.8 + The method should be used along with the ``wait_closed()`` method:: + + stream.close() + await stream.wait_closed() .. method:: can_write_eof() - Return *True* if the underlying transport supports - the :meth:`write_eof` method, *False* otherwise. + Return ``True`` if the underlying transport supports + the :meth:`write_eof` method, ``False`` otherwise. .. method:: write_eof() @@ -263,21 +280,6 @@ StreamWriter Access optional transport information; see :meth:`BaseTransport.get_extra_info` for details. - .. method:: write(data) - - Write *data* to the stream. - - This method is not subject to flow control. Calls to ``write()`` should - be followed by :meth:`drain`. The :meth:`awrite` method is a - recommended alternative the applies flow control automatically. - - .. method:: writelines(data) - - Write a list (or any iterable) of bytes to the stream. - - This method is not subject to flow control. Calls to ``writelines()`` - should be followed by :meth:`drain`. - .. coroutinemethod:: drain() Wait until it is appropriate to resume writing to the stream. @@ -293,10 +295,6 @@ StreamWriter be resumed. When there is nothing to wait for, the :meth:`drain` returns immediately. - .. method:: close() - - Close the stream. - .. method:: is_closing() Return ``True`` if the stream is closed or in the process of diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 00dc66c48b2189..eb1312a949a10a 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -6,6 +6,11 @@ Subprocesses ============ +**Source code:** :source:`Lib/asyncio/subprocess.py`, +:source:`Lib/asyncio/base_subprocess.py` + +---------------------------------------- + This section describes high-level async/await asyncio APIs to create and manage subprocesses. @@ -71,6 +76,10 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_exec` for other parameters. + .. deprecated-removed:: 3.8 3.10 + + The *loop* parameter. + .. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, \ stdout=None, stderr=None, loop=None, \ limit=None, \*\*kwds) @@ -86,14 +95,18 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. -.. important:: + .. important:: + + It is the application's responsibility to ensure that all whitespace and + special characters are quoted appropriately to avoid `shell injection + `_ + vulnerabilities. The :func:`shlex.quote` function can be used to properly + escape whitespace and special shell characters in strings that are going + to be used to construct shell commands. - It is the application's responsibility to ensure that all whitespace and - special characters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and special shell characters in strings that are going - to be used to construct shell commands. + .. deprecated-removed:: 3.8 3.10 + + The *loop* parameter. .. note:: @@ -293,18 +306,26 @@ their completion. Subprocess and Threads ---------------------- -Standard asyncio event loop supports running subprocesses from -different threads, but there are limitations: +Standard asyncio event loop supports running subprocesses from different threads by +default. + +On Windows subprocesses are provided by :class:`ProactorEventLoop` only (default), +:class:`SelectorEventLoop` has no subprocess support. + +On UNIX *child watchers* are used for subprocess finish waiting, see +:ref:`asyncio-watchers` for more info. + + +.. versionchanged:: 3.8 -* An event loop must run in the main thread. + UNIX switched to use :class:`ThreadedChildWatcher` for spawning subprocesses from + different threads without any limitation. -* The child watcher must be instantiated in the main thread - before executing subprocesses from other threads. Call the - :func:`get_child_watcher` function in the main thread to instantiate - the child watcher. + Spawning a subprocess with *inactive* current child watcher raises + :exc:`RuntimeError`. -Note that alternative event loop implementations might not share -the above limitations; please refer to their documentation. +Note that alternative event loop implementations might have own limitations; +please refer to their documentation. .. seealso:: diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index e3f18ccb4341fe..84a52cb2d57571 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -6,6 +6,10 @@ Synchronization Primitives ========================== +**Source code:** :source:`Lib/asyncio/locks.py` + +----------------------------------------------- + asyncio synchronization primitives are designed to be similar to those of the :mod:`threading` module with two important caveats: @@ -59,6 +63,9 @@ Lock finally: lock.release() + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + .. coroutinemethod:: acquire() Acquire the lock. @@ -66,6 +73,13 @@ Lock This method waits until the lock is *unlocked*, sets it to *locked* and returns ``True``. + When more than one coroutine is blocked in :meth:`acquire` + waiting for the lock to be unlocked, only one coroutine + eventually proceeds. + + Acquiring a lock is *fair*: the coroutine that proceeds will be + the first coroutine that started waiting on the lock. + .. method:: release() Release the lock. @@ -94,6 +108,10 @@ Event :meth:`clear` method. The :meth:`wait` method blocks until the flag is set to *true*. The flag is set to *false* initially. + + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + .. _asyncio_example_sync_event: Example:: @@ -166,6 +184,10 @@ Condition ``None``. In the latter case a new Lock object is created automatically. + + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + The preferred way to use a Condition is an :keyword:`async with` statement:: @@ -262,6 +284,10 @@ Semaphore internal counter (``1`` by default). If the given value is less than ``0`` a :exc:`ValueError` is raised. + + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + The preferred way to use a Semaphore is an :keyword:`async with` statement:: @@ -315,11 +341,14 @@ BoundedSemaphore increases the internal counter above the initial *value*. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + --------- -.. deprecated:: 3.7 +.. versionchanged:: 3.9 Acquiring a lock using ``await lock`` or ``yield from lock`` and/or :keyword:`with` statement (``with await lock``, ``with (yield from - lock)``) is deprecated. Use ``async with lock`` instead. + lock)``) was removed. Use ``async with lock`` instead. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index a1297f5fb7fc7f..99f012540d989b 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -18,9 +18,9 @@ and Tasks. Coroutines ========== -Coroutines declared with async/await syntax is the preferred way of -writing asyncio applications. For example, the following snippet -of code (requires Python 3.7+) prints "hello", waits 1 second, +:term:`Coroutines ` declared with the async/await syntax is the +preferred way of writing asyncio applications. For example, the following +snippet of code (requires Python 3.7+) prints "hello", waits 1 second, and then prints "world":: >>> import asyncio @@ -40,7 +40,7 @@ be executed:: >>> main() -To actually run a coroutine asyncio provides three main mechanisms: +To actually run a coroutine, asyncio provides three main mechanisms: * The :func:`asyncio.run` function to run the top-level entry point "main()" function (see the above example.) @@ -212,9 +212,11 @@ Running an asyncio Program .. function:: run(coro, \*, debug=False) + Execute the :term:`coroutine` *coro* and return the result. + This function runs the passed coroutine, taking care of - managing the asyncio event loop and *finalizing asynchronous - generators*. + managing the asyncio event loop, *finalizing asynchronous + generators*, and closing the threadpool. This function cannot be called when another asyncio event loop is running in the same thread. @@ -225,10 +227,22 @@ Running an asyncio Program the end. It should be used as a main entry point for asyncio programs, and should ideally only be called once. + Example:: + + async def main(): + await asyncio.sleep(1) + print('hello') + + asyncio.run(main()) + .. versionadded:: 3.7 - **Important:** this function has been added to asyncio in - Python 3.7 on a :term:`provisional basis `. + .. versionchanged:: 3.9 + Updated to use :meth:`loop.shutdown_default_executor`. + + .. note:: + The source code for ``asyncio.run()`` can be found in + :source:`Lib/asyncio/runners.py`. Creating Tasks ============== @@ -279,8 +293,8 @@ Sleeping ``sleep()`` always suspends the current task, allowing other tasks to run. - The *loop* argument is deprecated and scheduled for removal - in Python 3.10. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. .. _asyncio_example_sleep: @@ -334,6 +348,9 @@ Running Tasks Concurrently cancellation of one submitted Task/Future to cause other Tasks/Futures to be cancelled. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + .. _asyncio_example_gather: Example:: @@ -370,6 +387,14 @@ Running Tasks Concurrently # Task C: Compute factorial(4)... # Task C: factorial(4) = 24 + .. note:: + If *return_exceptions* is False, cancelling gather() after it + has been marked done won't cancel any submitted awaitables. + For instance, gather can be marked done after propagating an + exception to the caller, therefore, calling ``gather.cancel()`` + after catching an exception (raised by one of the awaitables) from + gather won't cancel any other awaitables. + .. versionchanged:: 3.7 If the *gather* itself is cancelled, the cancellation is propagated regardless of *return_exceptions*. @@ -411,6 +436,9 @@ Shielding From Cancellation except CancelledError: res = None + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + Timeouts ======== @@ -433,12 +461,13 @@ Timeouts wrap it in :func:`shield`. The function will wait until the future is actually cancelled, - so the total wait time may exceed the *timeout*. + so the total wait time may exceed the *timeout*. If an exception + happens during cancellation, it is propagated. If the wait is cancelled, the future *aw* is also cancelled. - The *loop* argument is deprecated and scheduled for removal - in Python 3.10. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. .. _asyncio_example_waitfor: @@ -478,10 +507,7 @@ Waiting Primitives set concurrently and block until the condition specified by *return_when*. - If any awaitable in *aws* is a coroutine, it is automatically - scheduled as a Task. Passing coroutines objects to - ``wait()`` directly is deprecated as it leads to - :ref:`confusing behavior `. + The *aws* set must not be empty. Returns two sets of Tasks/Futures: ``(done, pending)``. @@ -489,9 +515,6 @@ Waiting Primitives done, pending = await asyncio.wait(aws) - The *loop* argument is deprecated and scheduled for removal - in Python 3.10. - *timeout* (a float or int), if specified, can be used to control the maximum number of seconds to wait before returning. @@ -523,6 +546,17 @@ Waiting Primitives Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the futures when a timeout occurs. + .. deprecated:: 3.8 + + If any awaitable in *aws* is a coroutine, it is automatically + scheduled as a Task. Passing coroutines objects to + ``wait()`` directly is deprecated as it leads to + :ref:`confusing behavior `. + + .. deprecated-removed:: 3.8 3.10 + + The *loop* parameter. + .. _asyncio_example_wait_coroutine: .. note:: @@ -550,6 +584,8 @@ Waiting Primitives if task in done: # Everything will work as expected now. + .. deprecated-removed:: 3.8 3.11 + Passing coroutine objects to ``wait()`` directly is deprecated. @@ -557,20 +593,82 @@ Waiting Primitives .. function:: as_completed(aws, \*, loop=None, timeout=None) Run :ref:`awaitable objects ` in the *aws* - set concurrently. Return an iterator of :class:`Future` objects. - Each Future object returned represents the earliest result - from the set of the remaining awaitables. + set concurrently. Return an iterator of coroutines. + Each coroutine returned can be awaited to get the earliest next + result from the set of the remaining awaitables. Raises :exc:`asyncio.TimeoutError` if the timeout occurs before all Futures are done. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + Example:: - for f in as_completed(aws): - earliest_result = await f + for coro in as_completed(aws): + earliest_result = await coro # ... +Running in Threads +================== + +.. coroutinefunction:: to_thread(func, /, \*args, \*\*kwargs) + + Asynchronously run function *func* in a separate thread. + + Any \*args and \*\*kwargs supplied for this function are directly passed + to *func*. Also, the current :class:`contextvars.Context` is propogated, + allowing context variables from the event loop thread to be accessed in the + separate thread. + + Return a coroutine that can be awaited to get the eventual result of *func*. + + This coroutine function is primarily intended to be used for executing + IO-bound functions/methods that would otherwise block the event loop if + they were ran in the main thread. For example:: + + def blocking_io(): + print(f"start blocking_io at {time.strftime('%X')}") + # Note that time.sleep() can be replaced with any blocking + # IO-bound operation, such as file operations. + time.sleep(1) + print(f"blocking_io complete at {time.strftime('%X')}") + + async def main(): + print(f"started main at {time.strftime('%X')}") + + await asyncio.gather( + asyncio.to_thread(blocking_io), + asyncio.sleep(1)) + + print(f"finished main at {time.strftime('%X')}") + + + asyncio.run(main()) + + # Expected output: + # + # started main at 19:50:53 + # start blocking_io at 19:50:53 + # blocking_io complete at 19:50:54 + # finished main at 19:50:54 + + Directly calling `blocking_io()` in any coroutine would block the event loop + for its duration, resulting in an additional 1 second of run time. Instead, + by using `asyncio.to_thread()`, we can run it in a separate thread without + blocking the event loop. + + .. note:: + + Due to the :term:`GIL`, `asyncio.to_thread()` can typically only be used + to make IO-bound functions non-blocking. However, for extension modules + that release the GIL or alternative Python implementations that don't + have one, `asyncio.to_thread()` can also be used for CPU-bound functions. + + .. versionadded:: 3.9 + + Scheduling From Other Threads ============================= @@ -690,7 +788,10 @@ Task Object .. versionchanged:: 3.8 Added the ``name`` parameter. - .. method:: cancel() + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + + .. method:: cancel(msg=None) Request the Task to be cancelled. @@ -705,6 +806,9 @@ Task Object suppressing cancellation completely is not common and is actively discouraged. + .. versionchanged:: 3.9 + Added the ``msg`` parameter. + .. _asyncio_example_task_cancel: The following example illustrates how coroutines can intercept @@ -838,6 +942,12 @@ Task Object The *file* argument is an I/O stream to which the output is written; by default output is written to :data:`sys.stderr`. + .. method:: get_coro() + + Return the coroutine object wrapped by the :class:`Task`. + + .. versionadded:: 3.8 + .. method:: get_name() Return the name of the Task. @@ -860,28 +970,6 @@ Task Object .. versionadded:: 3.8 - .. classmethod:: all_tasks(loop=None) - - Return a set of all tasks for an event loop. - - By default all tasks for the current event loop are returned. - If *loop* is ``None``, the :func:`get_event_loop` function - is used to get the current loop. - - This method is **deprecated** and will be removed in - Python 3.9. Use the :func:`asyncio.all_tasks` function instead. - - .. classmethod:: current_task(loop=None) - - Return the currently running task or ``None``. - - If *loop* is ``None``, the :func:`get_event_loop` function - is used to get the current loop. - - This method is **deprecated** and will be removed in - Python 3.9. Use the :func:`asyncio.current_task` function - instead. - .. _asyncio_generator_based_coro: @@ -916,12 +1004,13 @@ enforced. async def main(): await old_style_coroutine() - This decorator is **deprecated** and is scheduled for removal in - Python 3.10. - This decorator should not be used for :keyword:`async def` coroutines. + .. deprecated-removed:: 3.8 3.10 + + Use :keyword:`async def` instead. + .. function:: iscoroutine(obj) Return ``True`` if *obj* is a :ref:`coroutine object `. diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 6990adb21e3603..94a853259d3483 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -4,7 +4,7 @@ .. module:: asyncio :synopsis: Asynchronous I/O. --------------- +------------------------------- .. sidebar:: Hello World! @@ -91,3 +91,6 @@ Additionally, there are **low-level** APIs for asyncio-api-index.rst asyncio-llapi-index.rst asyncio-dev.rst + +.. note:: + The source code for asyncio can be found in :source:`Lib/asyncio/`. diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst new file mode 100644 index 00000000000000..3c68a1515b3eff --- /dev/null +++ b/Doc/library/audit_events.rst @@ -0,0 +1,21 @@ +.. _audit-events: + +.. index:: single: audit events + +Audit events table +================== + +This table contains all events raised by :func:`sys.audit` or +:c:func:`PySys_Audit` calls throughout the CPython runtime and the +standard library. These calls were added in 3.8.0 or later. + +See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for +information on handling these events. + +.. impl-detail:: + + This table is generated from the CPython documentation, and may not + represent events raised by other implementations. See your runtime + specific documentation for actual events raised. + +.. audit-event-table:: diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index ad9f5f58bee2aa..2f24bb63912fb6 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -124,7 +124,7 @@ The modern interface provides: whether a lowercase alphabet is acceptable as input. For security purposes, the default is ``False``. - :rfc:`3548` allows for optional mapping of the digit 0 (zero) to the letter O + :rfc:`4648` allows for optional mapping of the digit 0 (zero) to the letter O (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) or letter L (el). The optional argument *map01* when not ``None``, specifies which letter the digit 1 should be mapped to (when *map01* is not ``None``, the @@ -136,6 +136,27 @@ The modern interface provides: input. +.. function:: b32hexencode(s) + + Similar to :func:`b32encode` but uses the Extended Hex Alphabet, as defined in + :rfc:`4648`. + + .. versionadded:: 3.10 + + +.. function:: b32hexdecode(s, casefold=False) + + Similar to :func:`b32decode` but uses the Extended Hex Alphabet, as defined in + :rfc:`4648`. + + This version does not allow the digit 0 (zero) to the letter O (oh) and digit + 1 (one) to either the letter I (eye) or letter L (el) mappings, all these + characters are included in the Extended Hex Alphabet and are not + interchangable. + + .. versionadded:: 3.10 + + .. function:: b16encode(s) Encode the :term:`bytes-like object` *s* using Base16 and return the @@ -235,12 +256,6 @@ The legacy interface: .. versionadded:: 3.1 -.. function:: decodestring(s) - - Deprecated alias of :func:`decodebytes`. - - .. deprecated:: 3.1 - .. function:: encode(input, output) @@ -261,12 +276,6 @@ The legacy interface: .. versionadded:: 3.1 -.. function:: encodestring(s) - - Deprecated alias of :func:`encodebytes`. - - .. deprecated:: 3.1 - An example usage of the module: diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 116ffcf88e3b69..7e4066cd436ad5 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -343,7 +343,7 @@ The :mod:`bdb` module also defines two classes: For backwards compatibility. Calls the :meth:`run` method. - .. method:: runcall(func, *args, **kwds) + .. method:: runcall(func, /, *args, **kwds) Debug a single function call, and return its result. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 89ecddc7780fa3..2c0c1bce5d7f8f 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -92,6 +92,8 @@ The :mod:`binascii` module defines the following functions: The string should contain a complete number of binary bytes, or (in case of the last portion of the binhex4 data) have the remaining bits zero. + .. deprecated:: 3.9 + .. function:: rledecode_hqx(data) @@ -104,11 +106,15 @@ The :mod:`binascii` module defines the following functions: .. versionchanged:: 3.2 Accept only bytestring or bytearray objects as input. + .. deprecated:: 3.9 + .. function:: rlecode_hqx(data) Perform binhex4 style RLE-compression on *data* and return the result. + .. deprecated:: 3.9 + .. function:: b2a_hqx(data) @@ -116,6 +122,8 @@ The :mod:`binascii` module defines the following functions: argument should already be RLE-coded, and have a length divisible by 3 (except possibly the last fragment). + .. deprecated:: 3.9 + .. function:: crc_hqx(data, value) @@ -145,8 +153,8 @@ The :mod:`binascii` module defines the following functions: platforms, use ``crc32(data) & 0xffffffff``. -.. function:: b2a_hex(data) - hexlify(data) +.. function:: b2a_hex(data[, sep[, bytes_per_sep=1]]) + hexlify(data[, sep[, bytes_per_sep=1]]) Return the hexadecimal representation of the binary *data*. Every byte of *data* is converted into the corresponding 2-digit hex representation. The @@ -155,6 +163,24 @@ The :mod:`binascii` module defines the following functions: Similar functionality (but returning a text string) is also conveniently accessible using the :meth:`bytes.hex` method. + If *sep* is specified, it must be a single character str or bytes object. + It will be inserted in the output after every *bytes_per_sep* input bytes. + Separator placement is counted from the right end of the output by default, + if you wish to count from the left, supply a negative *bytes_per_sep* value. + + >>> import binascii + >>> binascii.b2a_hex(b'\xb9\x01\xef') + b'b901ef' + >>> binascii.hexlify(b'\xb9\x01\xef', '-') + b'b9-01-ef' + >>> binascii.b2a_hex(b'\xb9\x01\xef', b'_', 2) + b'b9_01ef' + >>> binascii.b2a_hex(b'\xb9\x01\xef', b' ', -2) + b'b901 ef' + + .. versionchanged:: 3.8 + The *sep* and *bytes_per_sep* parameters were added. + .. function:: a2b_hex(hexstr) unhexlify(hexstr) diff --git a/Doc/library/binhex.rst b/Doc/library/binhex.rst index 2966e0dbfbcfe8..7de6a663762f0c 100644 --- a/Doc/library/binhex.rst +++ b/Doc/library/binhex.rst @@ -6,6 +6,8 @@ **Source code:** :source:`Lib/binhex.py` +.. deprecated:: 3.9 + -------------- This module encodes and decodes files in binhex4 format, a format allowing diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 6bf7814b257f4a..6666d55abe2e50 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -60,7 +60,7 @@ The following functions are provided: .. seealso:: `SortedCollection recipe - `_ that uses + `_ that uses bisect to build a full-featured collection class with straight-forward search methods and support for a key-function. The keys are precomputed to save unnecessary calls to the key function during searches. diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 946cc67dd301fc..85cdc16a7d78d4 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -31,7 +31,7 @@ All of the classes in this module may safely be accessed from multiple threads. (De)compression of files ------------------------ -.. function:: open(filename, mode='r', compresslevel=9, encoding=None, errors=None, newline=None) +.. function:: open(filename, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None) Open a bzip2-compressed file in binary or text mode, returning a :term:`file object`. @@ -65,7 +65,7 @@ All of the classes in this module may safely be accessed from multiple threads. Accepts a :term:`path-like object`. -.. class:: BZ2File(filename, mode='r', buffering=None, compresslevel=9) +.. class:: BZ2File(filename, mode='r', *, compresslevel=9) Open a bzip2-compressed file in binary mode. @@ -81,9 +81,7 @@ All of the classes in this module may safely be accessed from multiple threads. If *filename* is a file object (rather than an actual file name), a mode of ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``. - The *buffering* argument is ignored. Its use is deprecated since Python 3.0. - - If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be a number between + If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be an integer between ``1`` and ``9`` specifying the level of compression: ``1`` produces the least compression, and ``9`` (default) produces the most compression. @@ -110,9 +108,6 @@ All of the classes in this module may safely be accessed from multiple threads. .. versionadded:: 3.3 - .. deprecated:: 3.0 - The keyword argument *buffering* was deprecated and is now ignored. - .. versionchanged:: 3.1 Support for the :keyword:`with` statement was added. @@ -138,6 +133,13 @@ All of the classes in this module may safely be accessed from multiple threads. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.9 + The *buffering* parameter has been removed. It was ignored and deprecated + since Python 3.0. Pass an open file object to control how the file is + opened. + + The *compresslevel* parameter became keyword-only. + Incremental (de)compression --------------------------- @@ -148,7 +150,7 @@ Incremental (de)compression incrementally. For one-shot compression, use the :func:`compress` function instead. - *compresslevel*, if given, must be a number between ``1`` and ``9``. The + *compresslevel*, if given, must be an integer between ``1`` and ``9``. The default is ``9``. .. method:: compress(data) @@ -234,9 +236,9 @@ One-shot (de)compression .. function:: compress(data, compresslevel=9) - Compress *data*. + Compress *data*, a :term:`bytes-like object `. - *compresslevel*, if given, must be a number between ``1`` and ``9``. The + *compresslevel*, if given, must be an integer between ``1`` and ``9``. The default is ``9``. For incremental compression, use a :class:`BZ2Compressor` instead. @@ -244,7 +246,7 @@ One-shot (de)compression .. function:: decompress(data) - Decompress *data*. + Decompress *data*, a :term:`bytes-like object `. If *data* is the concatenation of multiple compressed streams, decompress all of the streams. @@ -254,3 +256,77 @@ One-shot (de)compression .. versionchanged:: 3.3 Support for multi-stream inputs was added. +.. _bz2-usage-examples: + +Examples of usage +----------------- + +Below are some examples of typical usage of the :mod:`bz2` module. + +Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: + + >>> import bz2 + + >>> data = b"""\ + ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue + ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, + ... sit amet cursus ante. In interdum laoreet mi, sit amet ultrices purus + ... pulvinar a. Nam gravida euismod magna, non varius justo tincidunt feugiat. + ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo + ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum + ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" + + >>> c = bz2.compress(data) + >>> len(data) / len(c) # Data compression ratio + 1.513595166163142 + + >>> d = bz2.decompress(c) + >>> data == d # Check equality to original object after round-trip + True + +Using :class:`BZ2Compressor` for incremental compression: + + >>> import bz2 + + >>> def gen_data(chunks=10, chunksize=1000): + ... """Yield incremental blocks of chunksize bytes.""" + ... for _ in range(chunks): + ... yield b"z" * chunksize + ... + >>> comp = bz2.BZ2Compressor() + >>> out = b"" + >>> for chunk in gen_data(): + ... # Provide data to the compressor object + ... out = out + comp.compress(chunk) + ... + >>> # Finish the compression process. Call this once you have + >>> # finished providing data to the compressor. + >>> out = out + comp.flush() + +The example above uses a very "nonrandom" stream of data +(a stream of `b"z"` chunks). Random data tends to compress poorly, +while ordered, repetitive data usually yields a high compression ratio. + +Writing and reading a bzip2-compressed file in binary mode: + + >>> import bz2 + + >>> data = b"""\ + ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue + ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, + ... sit amet cursus ante. In interdum laoreet mi, sit amet ultrices purus + ... pulvinar a. Nam gravida euismod magna, non varius justo tincidunt feugiat. + ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo + ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum + ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" + + >>> with bz2.open("myfile.bz2", "wb") as f: + ... # Write compressed data to file + ... unused = f.write(data) + + >>> with bz2.open("myfile.bz2", "rb") as f: + ... # Decompress data from file + ... content = f.read() + + >>> content == data # Check equality to original object after round-trip + True diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 56b75ef0f850a6..c3c04db853ed2d 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -349,7 +349,7 @@ For simple text calendars this module provides the following functions. .. function:: monthcalendar(year, month) Returns a matrix representing a month's calendar. Each row represents a week; - days outside of the month a represented by zeros. Each week begins with Monday + days outside of the month are represented by zeros. Each week begins with Monday unless set by :func:`setfirstweekday`. diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 9d81730f201222..28cd96b0e12da9 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -6,13 +6,12 @@ -------------- -This module is always available. It provides access to mathematical functions -for complex numbers. The functions in this module accept integers, -floating-point numbers or complex numbers as arguments. They will also accept -any Python object that has either a :meth:`__complex__` or a :meth:`__float__` -method: these methods are used to convert the object to a complex or -floating-point number, respectively, and the function is then applied to the -result of the conversion. +This module provides access to mathematical functions for complex numbers. The +functions in this module accept integers, floating-point numbers or complex +numbers as arguments. They will also accept any Python object that has either a +:meth:`__complex__` or a :meth:`__float__` method: these methods are used to +convert the object to a complex or floating-point number, respectively, and +the function is then applied to the result of the conversion. .. note:: diff --git a/Doc/library/code.rst b/Doc/library/code.rst index e2c47bab5a0b91..538e5afc7822aa 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -56,8 +56,8 @@ build applications which provide an interactive interpreter prompt. *source* is the source string; *filename* is the optional filename from which source was read, defaulting to ``''``; and *symbol* is the optional - grammar start symbol, which should be either ``'single'`` (the default) or - ``'eval'``. + grammar start symbol, which should be ``'single'`` (the default), ``'eval'`` + or ``'exec'``. Returns a code object (the same as ``compile(source, filename, symbol)``) if the command is complete and valid; ``None`` if the command is incomplete; raises @@ -76,7 +76,7 @@ Interactive Interpreter Objects Compile and run some source in the interpreter. Arguments are the same as for :func:`compile_command`; the default for *filename* is ``''``, and for - *symbol* is ``'single'``. One several things can happen: + *symbol* is ``'single'``. One of several things can happen: * The input is incorrect; :func:`compile_command` raised an exception (:exc:`SyntaxError` or :exc:`OverflowError`). A syntax traceback will be diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index b3246376846dd0..f071057293eece 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -292,7 +292,7 @@ Error Handlers To simplify and standardize error handling, codecs may implement different error handling schemes by -accepting the *errors* string argument. The following string values are +accepting the *errors* string argument. The following string values are defined and implemented by all standard Python codecs: .. tabularcolumns:: |l|L| @@ -301,11 +301,11 @@ defined and implemented by all standard Python codecs: | Value | Meaning | +=========================+===============================================+ | ``'strict'`` | Raise :exc:`UnicodeError` (or a subclass); | -| | this is the default. Implemented in | +| | this is the default. Implemented in | | | :func:`strict_errors`. | +-------------------------+-----------------------------------------------+ | ``'ignore'`` | Ignore the malformed data and continue | -| | without further notice. Implemented in | +| | without further notice. Implemented in | | | :func:`ignore_errors`. | +-------------------------+-----------------------------------------------+ @@ -327,11 +327,11 @@ The following error handlers are only applicable to | | marker; Python will use the official | | | ``U+FFFD`` REPLACEMENT CHARACTER for the | | | built-in codecs on decoding, and '?' on | -| | encoding. Implemented in | +| | encoding. Implemented in | | | :func:`replace_errors`. | +-------------------------+-----------------------------------------------+ | ``'xmlcharrefreplace'`` | Replace with the appropriate XML character | -| | reference (only for encoding). Implemented | +| | reference (only for encoding). Implemented | | | in :func:`xmlcharrefreplace_errors`. | +-------------------------+-----------------------------------------------+ | ``'backslashreplace'`` | Replace with backslashed escape sequences. | @@ -339,15 +339,15 @@ The following error handlers are only applicable to | | :func:`backslashreplace_errors`. | +-------------------------+-----------------------------------------------+ | ``'namereplace'`` | Replace with ``\N{...}`` escape sequences | -| | (only for encoding). Implemented in | +| | (only for encoding). Implemented in | | | :func:`namereplace_errors`. | +-------------------------+-----------------------------------------------+ | ``'surrogateescape'`` | On decoding, replace byte with individual | | | surrogate code ranging from ``U+DC80`` to | -| | ``U+DCFF``. This code will then be turned | +| | ``U+DCFF``. This code will then be turned | | | back into the same byte when the | | | ``'surrogateescape'`` error handler is used | -| | when encoding the data. (See :pep:`383` for | +| | when encoding the data. (See :pep:`383` for | | | more.) | +-------------------------+-----------------------------------------------+ @@ -357,7 +357,7 @@ In addition, the following error handler is specific to the given codecs: | Value | Codecs | Meaning | +===================+========================+===========================================+ |``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding of surrogate | -| | utf-16-be, utf-16-le, | codes. These codecs normally treat the | +| | utf-16-be, utf-16-le, | codes. These codecs normally treat the | | | utf-32-be, utf-32-le | presence of surrogates as an error. | +-------------------+------------------------+-------------------------------------------+ @@ -388,9 +388,9 @@ handler: error handler must either raise this or a different exception, or return a tuple with a replacement for the unencodable part of the input and a position where encoding should continue. The replacement may be either :class:`str` or - :class:`bytes`. If the replacement is bytes, the encoder will simply copy + :class:`bytes`. If the replacement is bytes, the encoder will simply copy them into the output buffer. If the replacement is a string, the encoder will - encode the replacement. Encoding continues on original input at the + encode the replacement. Encoding continues on original input at the specified position. Negative position values will be treated as being relative to the end of the input string. If the resulting position is out of bound an :exc:`IndexError` will be raised. @@ -484,7 +484,7 @@ function interfaces of the stateless encoder and decoder: .. method:: Codec.decode(input[, errors]) Decodes the object *input* and returns a tuple (output object, length - consumed). For instance, for a :term:`text encoding`, decoding converts + consumed). For instance, for a :term:`text encoding`, decoding converts a bytes object encoded using a particular character set encoding to a string object. @@ -568,7 +568,7 @@ define in order to be compatible with the Python codec registry. implementation should make sure that ``0`` is the most common state. (States that are more complicated than integers can be converted into an integer by marshaling/pickling the state and encoding the bytes - of the resulting string into an integer). + of the resulting string into an integer.) .. method:: setstate(state) @@ -751,7 +751,7 @@ compatible with the Python codec registry. number of encoded bytes or code points to read for decoding. The decoder can modify this setting as appropriate. The default value -1 indicates to read and decode as much as - possible. This parameter is intended to + possible. This parameter is intended to prevent having to decode huge files in one step. The *firstline* flag indicates that @@ -780,8 +780,8 @@ compatible with the Python codec registry. Read all lines available on the input stream and return them as a list of lines. - Line-endings are implemented using the codec's decoder method and are - included in the list entries if *keepends* is true. + Line-endings are implemented using the codec's :meth:`decode` method and + are included in the list entries if *keepends* is true. *sizehint*, if given, is passed as the *size* argument to the stream's :meth:`read` method. @@ -791,7 +791,7 @@ compatible with the Python codec registry. Resets the codec buffers used for keeping state. - Note that no stream repositioning should take place. This method is + Note that no stream repositioning should take place. This method is primarily intended to be able to recover from decoding errors. @@ -841,7 +841,7 @@ The design is such that one can use the factory functions returned by the code calling :meth:`read` and :meth:`write`, while *Reader* and *Writer* work on the backend — the data in *stream*. - You can use these objects to do transparent transcodings from e.g. Latin-1 + You can use these objects to do transparent transcodings, e.g., from Latin-1 to UTF-8 and back. The *stream* argument must be a file-like object. @@ -866,10 +866,10 @@ Encodings and Unicode --------------------- Strings are stored internally as sequences of code points in -range ``0x0``--``0x10FFFF``. (See :pep:`393` for +range ``0x0``--``0x10FFFF``. (See :pep:`393` for more details about the implementation.) Once a string object is used outside of CPU and memory, endianness -and how these arrays are stored as bytes become an issue. As with other +and how these arrays are stored as bytes become an issue. As with other codecs, serialising a string into a sequence of bytes is known as *encoding*, and recreating the string from the sequence of bytes is known as *decoding*. There are a variety of different text serialisation codecs, which are @@ -964,7 +964,7 @@ to determine the byte order used for generating the byte sequence, but as a signature that helps in guessing the encoding. On encoding the utf-8-sig codec will write ``0xef``, ``0xbb``, ``0xbf`` as the first three bytes to the file. On decoding ``utf-8-sig`` will skip those three bytes if they appear as the first -three bytes in the file. In UTF-8, the use of the BOM is discouraged and +three bytes in the file. In UTF-8, the use of the BOM is discouraged and should generally be avoided. @@ -984,7 +984,7 @@ e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. .. impl-detail:: Some common encodings can bypass the codecs lookup machinery to - improve performance. These optimization opportunities are only + improve performance. These optimization opportunities are only recognized by CPython for a limited set of (case insensitive) aliases: utf-8, utf8, latin-1, latin1, iso-8859-1, iso8859-1, mbcs (Windows only), ascii, us-ascii, utf-16, utf16, utf-32, utf32, and @@ -1106,11 +1106,6 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp1258 | windows-1258 | Vietnamese | +-----------------+--------------------------------+--------------------------------+ -| cp65001 | | Windows only: Windows UTF-8 | -| | | (``CP_UTF8``) | -| | | | -| | | .. versionadded:: 3.3 | -+-----------------+--------------------------------+--------------------------------+ | euc_jp | eucjp, ujis, u-jis | Japanese | +-----------------+--------------------------------+--------------------------------+ | euc_jis_2004 | jisx0213, eucjis2004 | Japanese | @@ -1150,7 +1145,7 @@ particular, the following variants typically exist: | iso2022_kr | csiso2022kr, iso2022kr, | Korean | | | iso-2022-kr | | +-----------------+--------------------------------+--------------------------------+ -| latin_1 | iso-8859-1, iso8859-1, 8859, | West Europe | +| latin_1 | iso-8859-1, iso8859-1, 8859, | Western Europe | | | cp819, latin, latin1, L1 | | +-----------------+--------------------------------+--------------------------------+ | iso8859_2 | iso-8859-2, latin2, L2 | Central and Eastern Europe | @@ -1203,7 +1198,8 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | mac_iceland | maciceland | Icelandic | +-----------------+--------------------------------+--------------------------------+ -| mac_latin2 | maclatin2, maccentraleurope | Central and Eastern Europe | +| mac_latin2 | maclatin2, maccentraleurope, | Central and Eastern Europe | +| | mac_centeuro | | +-----------------+--------------------------------+--------------------------------+ | mac_roman | macroman, macintosh | Western Europe | +-----------------+--------------------------------+--------------------------------+ @@ -1235,7 +1231,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | utf_7 | U7, unicode-1-1-utf-7 | all languages | +-----------------+--------------------------------+--------------------------------+ -| utf_8 | U8, UTF, utf8 | all languages | +| utf_8 | U8, UTF, utf8, cp65001 | all languages | +-----------------+--------------------------------+--------------------------------+ | utf_8_sig | | all languages | +-----------------+--------------------------------+--------------------------------+ @@ -1246,16 +1242,19 @@ particular, the following variants typically exist: The utf-32\* decoders no longer decode byte sequences that correspond to surrogate code points. +.. versionchanged:: 3.8 + ``cp65001`` is now an alias to ``utf_8``. + Python Specific Encodings ------------------------- A number of predefined codecs are specific to Python, so their codec names have -no meaning outside Python. These are listed in the tables below based on the +no meaning outside Python. These are listed in the tables below based on the expected input and output types (note that while text encodings are the most common use case for codecs, the underlying codec infrastructure supports -arbitrary data transforms rather than just text encodings). For asymmetric -codecs, the stated purpose describes the encoding direction. +arbitrary data transforms rather than just text encodings). For asymmetric +codecs, the stated meaning describes the encoding direction. Text Encodings ^^^^^^^^^^^^^^ @@ -1267,27 +1266,27 @@ encodings. .. tabularcolumns:: |l|p{0.3\linewidth}|p{0.3\linewidth}| +--------------------+---------+---------------------------+ -| Codec | Aliases | Purpose | +| Codec | Aliases | Meaning | +====================+=========+===========================+ -| idna | | Implements :rfc:`3490`, | +| idna | | Implement :rfc:`3490`, | | | | see also | | | | :mod:`encodings.idna`. | | | | Only ``errors='strict'`` | | | | is supported. | +--------------------+---------+---------------------------+ -| mbcs | ansi, | Windows only: Encode | +| mbcs | ansi, | Windows only: Encode the | | | dbcs | operand according to the | -| | | ANSI codepage (CP_ACP) | +| | | ANSI codepage (CP_ACP). | +--------------------+---------+---------------------------+ -| oem | | Windows only: Encode | +| oem | | Windows only: Encode the | | | | operand according to the | -| | | OEM codepage (CP_OEMCP) | +| | | OEM codepage (CP_OEMCP). | | | | | | | | .. versionadded:: 3.6 | +--------------------+---------+---------------------------+ -| palmos | | Encoding of PalmOS 3.5 | +| palmos | | Encoding of PalmOS 3.5. | +--------------------+---------+---------------------------+ -| punycode | | Implements :rfc:`3492`. | +| punycode | | Implement :rfc:`3492`. | | | | Stateful codecs are not | | | | supported. | +--------------------+---------+---------------------------+ @@ -1310,8 +1309,8 @@ encodings. | | | literal in ASCII-encoded | | | | Python source code, | | | | except that quotes are | -| | | not escaped. Decodes from | -| | | Latin-1 source code. | +| | | not escaped. Decode | +| | | from Latin-1 source code. | | | | Beware that Python source | | | | code actually uses UTF-8 | | | | by default. | @@ -1327,19 +1326,19 @@ Binary Transforms ^^^^^^^^^^^^^^^^^ The following codecs provide binary transforms: :term:`bytes-like object` -to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` +to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` (which only produces :class:`str` output). .. tabularcolumns:: |l|L|L|L| +----------------------+------------------+------------------------------+------------------------------+ -| Codec | Aliases | Purpose | Encoder / decoder | +| Codec | Aliases | Meaning | Encoder / decoder | +======================+==================+==============================+==============================+ -| base64_codec [#b64]_ | base64, base_64 | Convert operand to multiline | :meth:`base64.encodebytes` / | -| | | MIME base64 (the result | :meth:`base64.decodebytes` | -| | | always includes a trailing | | -| | | ``'\n'``) | | +| base64_codec [#b64]_ | base64, base_64 | Convert the operand to | :meth:`base64.encodebytes` / | +| | | multiline MIME base64 (the | :meth:`base64.decodebytes` | +| | | result always includes a | | +| | | trailing ``'\n'``). | | | | | | | | | | .. versionchanged:: 3.4 | | | | | accepts any | | @@ -1347,23 +1346,23 @@ to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` | | | as input for encoding and | | | | | decoding | | +----------------------+------------------+------------------------------+------------------------------+ -| bz2_codec | bz2 | Compress the operand | :meth:`bz2.compress` / | -| | | using bz2 | :meth:`bz2.decompress` | +| bz2_codec | bz2 | Compress the operand using | :meth:`bz2.compress` / | +| | | bz2. | :meth:`bz2.decompress` | +----------------------+------------------+------------------------------+------------------------------+ -| hex_codec | hex | Convert operand to | :meth:`binascii.b2a_hex` / | +| hex_codec | hex | Convert the operand to | :meth:`binascii.b2a_hex` / | | | | hexadecimal | :meth:`binascii.a2b_hex` | | | | representation, with two | | -| | | digits per byte | | +| | | digits per byte. | | +----------------------+------------------+------------------------------+------------------------------+ -| quopri_codec | quopri, | Convert operand to MIME | :meth:`quopri.encode` with | -| | quotedprintable, | quoted printable | ``quotetabs=True`` / | +| quopri_codec | quopri, | Convert the operand to MIME | :meth:`quopri.encode` with | +| | quotedprintable, | quoted printable. | ``quotetabs=True`` / | | | quoted_printable | | :meth:`quopri.decode` | +----------------------+------------------+------------------------------+------------------------------+ | uu_codec | uu | Convert the operand using | :meth:`uu.encode` / | -| | | uuencode | :meth:`uu.decode` | +| | | uuencode. | :meth:`uu.decode` | +----------------------+------------------+------------------------------+------------------------------+ -| zlib_codec | zip, zlib | Compress the operand | :meth:`zlib.compress` / | -| | | using gzip | :meth:`zlib.decompress` | +| zlib_codec | zip, zlib | Compress the operand using | :meth:`zlib.compress` / | +| | | gzip. | :meth:`zlib.decompress` | +----------------------+------------------+------------------------------+------------------------------+ .. [#b64] In addition to :term:`bytes-like objects `, @@ -1383,16 +1382,17 @@ Text Transforms ^^^^^^^^^^^^^^^ The following codec provides a text transform: a :class:`str` to :class:`str` -mapping. It is not supported by :meth:`str.encode` (which only produces +mapping. It is not supported by :meth:`str.encode` (which only produces :class:`bytes` output). .. tabularcolumns:: |l|l|L| +--------------------+---------+---------------------------+ -| Codec | Aliases | Purpose | +| Codec | Aliases | Meaning | +====================+=========+===========================+ -| rot_13 | rot13 | Returns the Caesar-cypher | -| | | encryption of the operand | +| rot_13 | rot13 | Return the Caesar-cypher | +| | | encryption of the | +| | | operand. | +--------------------+---------+---------------------------+ .. versionadded:: 3.2 @@ -1430,7 +1430,7 @@ conversion between Unicode and ACE, separating an input string into labels based on the separator characters defined in :rfc:`section 3.1 of RFC 3490 <3490#section-3.1>` and converting each label to ACE as required, and conversely separating an input byte string into labels based on the ``.`` separator and converting any ACE -labels found into unicode. Furthermore, the :mod:`socket` module +labels found into unicode. Furthermore, the :mod:`socket` module transparently converts Unicode host names to ACE, so that applications need not be concerned about converting host names themselves when they pass them to the socket module. On top of that, modules that have host names as function @@ -1439,7 +1439,7 @@ names (:mod:`http.client` then also transparently sends an IDNA hostname in the :mailheader:`Host` field if it sends that field at all). When receiving host names from the wire (such as in reverse name lookup), no -automatic conversion to Unicode is performed: Applications wishing to present +automatic conversion to Unicode is performed: applications wishing to present such host names to the user should decode them to Unicode. The module :mod:`encodings.idna` also implements the nameprep procedure, which @@ -1471,7 +1471,7 @@ functions can be used directly if desired. .. module:: encodings.mbcs :synopsis: Windows ANSI codepage -Encode operand according to the ANSI codepage (CP_ACP). +This module implements the ANSI codepage (CP_ACP). .. availability:: Windows only. @@ -1490,7 +1490,7 @@ Encode operand according to the ANSI codepage (CP_ACP). :synopsis: UTF-8 codec with BOM signature .. moduleauthor:: Walter Dörwald -This module implements a variant of the UTF-8 codec: On encoding a UTF-8 encoded +This module implements a variant of the UTF-8 codec. On encoding, a UTF-8 encoded BOM will be prepended to the UTF-8 encoded bytes. For the stateful encoder this -is only done once (on the first write to the byte stream). For decoding an +is only done once (on the first write to the byte stream). On decoding, an optional UTF-8 encoded BOM at the start of the data will be skipped. diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index a52d2c62c4fea1..c66b9d3ec0a26d 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -43,8 +43,9 @@ To do just the former: :exc:`OverflowError` or :exc:`ValueError` if there is an invalid literal. The *symbol* argument determines whether *source* is compiled as a statement - (``'single'``, the default) or as an :term:`expression` (``'eval'``). Any - other value will cause :exc:`ValueError` to be raised. + (``'single'``, the default), as a sequence of statements (``'exec'``) or + as an :term:`expression` (``'eval'``). Any other value will + cause :exc:`ValueError` to be raised. .. note:: diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 2a3fb142f7297e..a6038098675da2 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -98,12 +98,20 @@ ABC Inherits from Abstract Methods Mixin .. class:: Container - Hashable - Sized - Callable - ABCs for classes that provide respectively the methods :meth:`__contains__`, - :meth:`__hash__`, :meth:`__len__`, and :meth:`__call__`. + ABC for classes that provide the :meth:`__contains__` method. + +.. class:: Hashable + + ABC for classes that provide the :meth:`__hash__` method. + +.. class:: Sized + + ABC for classes that provide the :meth:`__len__` method. + +.. class:: Callable + + ABC for classes that provide the :meth:`__call__` method. .. class:: Iterable @@ -185,7 +193,7 @@ ABC Inherits from Abstract Methods Mixin expressions. Custom implementations must provide the :meth:`__await__` method. - :term:`Coroutine` objects and instances of the + :term:`Coroutine ` objects and instances of the :class:`~collections.abc.Coroutine` ABC are all instances of this ABC. .. note:: @@ -300,7 +308,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: .. seealso:: - * `OrderedSet recipe `_ for an + * `OrderedSet recipe `_ for an example built on :class:`MutableSet`. * For more about ABCs, see the :mod:`abc` module and :pep:`3119`. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 64de970fec94b4..a7d01b3f397a70 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -33,10 +33,10 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`, :class:`UserString` wrapper around string objects for easier string subclassing ===================== ==================================================================== -.. versionchanged:: 3.3 +.. deprecated-removed:: 3.3 3.10 Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module. For backwards compatibility, they continue to be visible in this module through - Python 3.7. Subsequently, they will be removed entirely. + Python 3.9. :class:`ChainMap` objects @@ -116,6 +116,9 @@ The class can be used to simulate nested scopes and is useful in templating. >>> list(combined) ['music', 'art', 'opera'] + .. versionchanged:: 3.9 + Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. + .. seealso:: * The `MultiContext class @@ -132,12 +135,12 @@ The class can be used to simulate nested scopes and is useful in templating. :attr:`~collections.ChainMap.parents` property. * The `Nested Contexts recipe - `_ has options to control + `_ has options to control whether writes and other mutations apply only to the first mapping or to any mapping in the chain. * A `greatly simplified read-only version of Chainmap - `_. + `_. :class:`ChainMap` Examples and Recipes @@ -162,7 +165,7 @@ environment variables which in turn take precedence over default values:: parser.add_argument('-u', '--user') parser.add_argument('-c', '--color') namespace = parser.parse_args() - command_line_args = {k:v for k, v in vars(namespace).items() if v} + command_line_args = {k: v for k, v in vars(namespace).items() if v is not None} combined = ChainMap(command_line_args, os.environ, defaults) print(combined['color']) @@ -325,6 +328,19 @@ For example:: instead of replacing them. Also, the *iterable* is expected to be a sequence of elements, not a sequence of ``(key, value)`` pairs. +Counters support rich comparison operators for equality, subset, and +superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. +All of those tests treat missing elements as having zero counts so that +``Counter(a=1) == Counter(a=1, b=0)`` returns true. + +.. versionadded:: 3.10 + Rich comparison operations we were added + +.. versionchanged:: 3.10 + In equality tests, missing elements are treated as having zero counts. + Formerly, ``Counter(a=3)`` and ``Counter(a=3, b=0)`` were considered + distinct. + Common patterns for working with :class:`Counter` objects:: sum(c.values()) # total of all counts @@ -729,6 +745,10 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, initialized from the first argument to the constructor, if present, or to ``None``, if absent. + .. versionchanged:: 3.9 + Added merge (``|``) and update (``|=``) operators, specified in + :pep:`584`. + :class:`defaultdict` Examples ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1017,15 +1037,6 @@ fields: .. versionchanged:: 3.5 Property docstrings became writeable. -Default values can be implemented by using :meth:`~somenamedtuple._replace` to -customize a prototype instance: - - >>> Account = namedtuple('Account', 'owner balance transaction_count') - >>> default_account = Account('', 0.0, 0) - >>> johns_account = default_account._replace(owner='John') - >>> janes_account = default_account._replace(owner='Jane') - - .. seealso:: * See :class:`typing.NamedTuple` for a way to add type hints for named @@ -1128,6 +1139,10 @@ anywhere a regular dictionary is used. passed to the :class:`OrderedDict` constructor and its :meth:`update` method. +.. versionchanged:: 3.9 + Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. + + :class:`OrderedDict` Examples and Recipes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1141,7 +1156,7 @@ original insertion position is changed and moved to the end:: def __setitem__(self, key, value): super().__setitem__(key, value) - super().move_to_end(key) + self.move_to_end(key) An :class:`OrderedDict` would also be useful for implementing variants of :func:`functools.lru_cache`:: @@ -1149,7 +1164,7 @@ variants of :func:`functools.lru_cache`:: class LRU(OrderedDict): 'Limit size, evicting the least recently looked-up key when full' - def __init__(self, maxsize=128, *args, **kwds): + def __init__(self, maxsize=128, /, *args, **kwds): self.maxsize = maxsize super().__init__(*args, **kwds) @@ -1159,6 +1174,8 @@ variants of :func:`functools.lru_cache`:: return value def __setitem__(self, key, value): + if key in self: + self.move_to_end(key) super().__setitem__(key, value) if len(self) > self.maxsize: oldest = next(iter(self)) diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index 1360c7cc9f1db4..b672a05b39145d 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -21,7 +21,7 @@ spaces, the coordinates are all between 0 and 1. .. seealso:: More information about color spaces can be found at - http://poynton.ca/ColorFAQ.html and + https://poynton.ca/ColorFAQ.html and https://www.cambridgeincolour.com/tutorials/color-spaces.htm. The :mod:`colorsys` module defines the following functions: diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 5e08616e9347b0..9b914b1f0d9c6d 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -52,6 +52,13 @@ compile Python sources. cases where the source file does not exist at the time the byte-code file is executed. +.. cmdoption:: -s strip_prefix +.. cmdoption:: -p prepend_prefix + + Remove (``-s``) or append (``-p``) the given prefix of paths + recorded in the ``.pyc`` files. + Cannot be combined with ``-d``. + .. cmdoption:: -x regex regex is used to search the full path to each file considered for @@ -96,6 +103,21 @@ compile Python sources. variable is not set, and ``checked-hash`` if the ``SOURCE_DATE_EPOCH`` environment variable is set. +.. cmdoption:: -o level + + Compile with the given optimization level. May be used multiple times + to compile for multiple levels at a time (for example, + ``compileall -o 1 -o 2``). + +.. cmdoption:: -e dir + + Ignore symlinks pointing outside the given directory. + +.. cmdoption:: --hardlink-dupes + + If two ``.pyc`` files with different optimization level have + the same content, use hard links to consolidate duplicate files. + .. versionchanged:: 3.2 Added the ``-i``, ``-b`` and ``-h`` options. @@ -107,6 +129,12 @@ compile Python sources. .. versionchanged:: 3.7 Added the ``--invalidation-mode`` option. +.. versionchanged:: 3.9 + Added the ``-s``, ``-p``, ``-e`` and ``--hardlink-dupes`` options. + Raised the default recursion limit from 10 to + :py:func:`sys.getrecursionlimit()`. + Added the possibility to specify the ``-o`` option multiple times. + There is no command-line option to control the optimization level used by the :func:`compile` function, because the Python interpreter itself already @@ -120,14 +148,14 @@ runtime. Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) +.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, \*, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. Return a true value if all the files compiled successfully, and a false value otherwise. The *maxlevels* parameter is used to limit the depth of the recursion; it - defaults to ``10``. + defaults to ``sys.getrecursionlimit()``. If *ddir* is given, it is prepended to the path to each file being compiled for use in compilation time tracebacks, and is also compiled in to the @@ -153,18 +181,27 @@ Public functions coexist. *optimize* specifies the optimization level for the compiler. It is passed to - the built-in :func:`compile` function. + the built-in :func:`compile` function. Accepts also a sequence of optimization + levels which lead to multiple compilations of one :file:`.py` file in one call. The argument *workers* specifies how many workers are used to compile files in parallel. The default is to not use multiple workers. If the platform can't use multiple workers and *workers* argument is given, - then sequential compilation will be used as a fallback. If *workers* is + then sequential compilation will be used as a fallback. If *workers* + is 0, the number of cores in the system is used. If *workers* is lower than ``0``, a :exc:`ValueError` will be raised. *invalidation_mode* should be a member of the :class:`py_compile.PycInvalidationMode` enum and controls how the generated pycs are invalidated at runtime. + The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to + the ``-s``, ``-p`` and ``-e`` options described above. + They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + + If *hardlink_dupes* is true and two ``.pyc`` files with different optimization + level have the same content, use hard links to consolidate duplicate files. + .. versionchanged:: 3.2 Added the *legacy* and *optimize* parameter. @@ -184,7 +221,17 @@ Public functions .. versionchanged:: 3.7 The *invalidation_mode* parameter was added. -.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) + .. versionchanged:: 3.7.2 + The *invalidation_mode* parameter's default value is updated to None. + + .. versionchanged:: 3.8 + Setting *workers* to 0 now chooses the optimal number of cores. + + .. versionchanged:: 3.9 + Added *stripdir*, *prependdir*, *limit_sl_dest* and *hardlink_dupes* arguments. + Default value of *maxlevels* was changed from ``10`` to ``sys.getrecursionlimit()`` + +.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, \*, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) Compile the file with path *fullname*. Return a true value if the file compiled successfully, and a false value otherwise. @@ -210,12 +257,20 @@ Public functions coexist. *optimize* specifies the optimization level for the compiler. It is passed to - the built-in :func:`compile` function. + the built-in :func:`compile` function. Accepts also a sequence of optimization + levels which lead to multiple compilations of one :file:`.py` file in one call. *invalidation_mode* should be a member of the :class:`py_compile.PycInvalidationMode` enum and controls how the generated pycs are invalidated at runtime. + The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to + the ``-s``, ``-p`` and ``-e`` options described above. + They may be specified as ``str``, ``bytes`` or :py:class:`os.PathLike`. + + If *hardlink_dupes* is true and two ``.pyc`` files with different optimization + level have the same content, use hard links to consolidate duplicate files. + .. versionadded:: 3.2 .. versionchanged:: 3.5 @@ -228,7 +283,13 @@ Public functions .. versionchanged:: 3.7 The *invalidation_mode* parameter was added. -.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) + .. versionchanged:: 3.7.2 + The *invalidation_mode* parameter's default value is updated to None. + + .. versionchanged:: 3.9 + Added *stripdir*, *prependdir*, *limit_sl_dest* and *hardlink_dupes* arguments. + +.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=None) Byte-compile all the :file:`.py` files found along ``sys.path``. Return a true value if all the files compiled successfully, and a false value otherwise. @@ -251,6 +312,9 @@ Public functions .. versionchanged:: 3.7 The *invalidation_mode* parameter was added. + .. versionchanged:: 3.7.2 + The *invalidation_mode* parameter's default value is updated to None. + To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 39cd9ff4826597..b150990b83b75b 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -28,5 +28,3 @@ The following are support modules for some of the above services: .. toctree:: _thread.rst - _dummy_thread.rst - dummy_threading.rst diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 8d6b1e8d71ff47..675a9ffdd0711a 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -28,7 +28,7 @@ Executor Objects An abstract class that provides methods to execute calls asynchronously. It should not be used directly, but through its concrete subclasses. - .. method:: submit(fn, *args, **kwargs) + .. method:: submit(fn, /, *args, **kwargs) Schedules the callable, *fn*, to be executed as ``fn(*args **kwargs)`` and returns a :class:`Future` object representing the execution of the @@ -67,7 +67,7 @@ Executor Objects .. versionchanged:: 3.5 Added the *chunksize* argument. - .. method:: shutdown(wait=True) + .. method:: shutdown(wait=True, \*, cancel_futures=False) Signal the executor that it should free any resources that it is using when the currently pending futures are done executing. Calls to @@ -82,6 +82,15 @@ Executor Objects value of *wait*, the entire Python program will not exit until all pending futures are done executing. + If *cancel_futures* is ``True``, this method will cancel all pending + futures that the executor has not started running. Any futures that + are completed or running won't be cancelled, regardless of the value + of *cancel_futures*. + + If both *cancel_futures* and *wait* are ``True``, all futures that the + executor has started running will be completed prior to this method + returning. The remaining futures are cancelled. + You can avoid having to call this method explicitly if you use the :keyword:`with` statement, which will shutdown the :class:`Executor` (waiting as if :meth:`Executor.shutdown` were called with *wait* set to @@ -94,6 +103,9 @@ Executor Objects e.submit(shutil.copy, 'src3.txt', 'dest3.txt') e.submit(shutil.copy, 'src4.txt', 'dest4.txt') + .. versionchanged:: 3.9 + Added *cancel_futures*. + ThreadPoolExecutor ------------------ @@ -159,6 +171,15 @@ And:: .. versionchanged:: 3.7 Added the *initializer* and *initargs* arguments. + .. versionchanged:: 3.8 + Default value of *max_workers* is changed to ``min(32, os.cpu_count() + 4)``. + This default value preserves at least 5 workers for I/O bound tasks. + It utilizes at most 32 CPU cores for CPU bound tasks which release the GIL. + And it avoids using very large resources implicitly on many-core machines. + + ThreadPoolExecutor now reuses idle worker threads before starting + *max_workers* worker threads too. + .. _threadpoolexecutor-example: @@ -200,7 +221,8 @@ ProcessPoolExecutor The :class:`ProcessPoolExecutor` class is an :class:`Executor` subclass that uses a pool of processes to execute calls asynchronously. :class:`ProcessPoolExecutor` uses the :mod:`multiprocessing` module, which -allows it to side-step the :term:`Global Interpreter Lock` but also means that +allows it to side-step the :term:`Global Interpreter Lock +` but also means that only picklable objects can be executed and returned. The ``__main__`` module must be importable by worker subprocesses. This means @@ -216,6 +238,10 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. given, it will default to the number of processors on the machine. If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError` will be raised. + On Windows, *max_workers* must be equal or lower than ``61``. If it is not + then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then + the default chosen will be at most ``61``, even if more processors are + available. *mp_context* can be a multiprocessing context or None. It will be used to launch the workers. If *mp_context* is ``None`` or not given, the default multiprocessing context is used. @@ -293,9 +319,10 @@ The :class:`Future` class encapsulates the asynchronous execution of a callable. .. method:: cancel() - Attempt to cancel the call. If the call is currently being executed and - cannot be cancelled then the method will return ``False``, otherwise the - call will be cancelled and the method will return ``True``. + Attempt to cancel the call. If the call is currently being executed or + finished running and cannot be cancelled then the method will return + ``False``, otherwise the call will be cancelled and the method will + return ``True``. .. method:: cancelled() @@ -410,8 +437,9 @@ Module Functions Wait for the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* to complete. Returns a named 2-tuple of sets. The first set, named ``done``, contains the futures that - completed (finished or were cancelled) before the wait completed. The second - set, named ``not_done``, contains uncompleted futures. + completed (finished or cancelled futures) before the wait completed. The + second set, named ``not_done``, contains the futures that did not complete + (pending or running futures). *timeout* can be used to control the maximum number of seconds to wait before returning. *timeout* can be an int or float. If *timeout* is not specified @@ -442,7 +470,7 @@ Module Functions Returns an iterator over the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or were cancelled). Any futures given by *fs* that + they complete (finished or cancelled futures). Any futures given by *fs* that are duplicated will be returned once. Any futures that completed before :func:`as_completed` is called will be yielded first. The returned iterator raises a :exc:`concurrent.futures.TimeoutError` if :meth:`~iterator.__next__` diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 04b52dc7b21531..2e22a549ee2813 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -313,6 +313,8 @@ from ``get()`` calls. my_dir: %(home_dir)s/lumberjack my_pictures: %(my_dir)s/Pictures + [Escape] + gain: 80%% # use a %% to escape the % sign (% is the only character that needs to be escaped) In the example above, :class:`ConfigParser` with *interpolation* set to ``BasicInterpolation()`` would resolve ``%(home_dir)s`` to the value of @@ -346,6 +348,9 @@ from ``get()`` calls. my_dir: ${home_dir}/lumberjack my_pictures: ${my_dir}/Pictures + [Escape] + cost: $$80 # use a $$ to escape the $ sign ($ is the only character that needs to be escaped) + Values from other sections can be fetched as well: .. code-block:: ini @@ -669,97 +674,98 @@ be overridden by subclasses or by attribute assignment. .. attribute:: ConfigParser.BOOLEAN_STATES - By default when using :meth:`~ConfigParser.getboolean`, config parsers - consider the following values ``True``: ``'1'``, ``'yes'``, ``'true'``, - ``'on'`` and the following values ``False``: ``'0'``, ``'no'``, ``'false'``, - ``'off'``. You can override this by specifying a custom dictionary of strings - and their Boolean outcomes. For example: + By default when using :meth:`~ConfigParser.getboolean`, config parsers + consider the following values ``True``: ``'1'``, ``'yes'``, ``'true'``, + ``'on'`` and the following values ``False``: ``'0'``, ``'no'``, ``'false'``, + ``'off'``. You can override this by specifying a custom dictionary of strings + and their Boolean outcomes. For example: - .. doctest:: + .. doctest:: - >>> custom = configparser.ConfigParser() - >>> custom['section1'] = {'funky': 'nope'} - >>> custom['section1'].getboolean('funky') - Traceback (most recent call last): - ... - ValueError: Not a boolean: nope - >>> custom.BOOLEAN_STATES = {'sure': True, 'nope': False} - >>> custom['section1'].getboolean('funky') - False + >>> custom = configparser.ConfigParser() + >>> custom['section1'] = {'funky': 'nope'} + >>> custom['section1'].getboolean('funky') + Traceback (most recent call last): + ... + ValueError: Not a boolean: nope + >>> custom.BOOLEAN_STATES = {'sure': True, 'nope': False} + >>> custom['section1'].getboolean('funky') + False - Other typical Boolean pairs include ``accept``/``reject`` or - ``enabled``/``disabled``. + Other typical Boolean pairs include ``accept``/``reject`` or + ``enabled``/``disabled``. .. method:: ConfigParser.optionxform(option) + :noindex: - This method transforms option names on every read, get, or set - operation. The default converts the name to lowercase. This also - means that when a configuration file gets written, all keys will be - lowercase. Override this method if that's unsuitable. - For example: + This method transforms option names on every read, get, or set + operation. The default converts the name to lowercase. This also + means that when a configuration file gets written, all keys will be + lowercase. Override this method if that's unsuitable. + For example: - .. doctest:: + .. doctest:: + + >>> config = """ + ... [Section1] + ... Key = Value + ... + ... [Section2] + ... AnotherKey = Value + ... """ + >>> typical = configparser.ConfigParser() + >>> typical.read_string(config) + >>> list(typical['Section1'].keys()) + ['key'] + >>> list(typical['Section2'].keys()) + ['anotherkey'] + >>> custom = configparser.RawConfigParser() + >>> custom.optionxform = lambda option: option + >>> custom.read_string(config) + >>> list(custom['Section1'].keys()) + ['Key'] + >>> list(custom['Section2'].keys()) + ['AnotherKey'] - >>> config = """ - ... [Section1] - ... Key = Value - ... - ... [Section2] - ... AnotherKey = Value - ... """ - >>> typical = configparser.ConfigParser() - >>> typical.read_string(config) - >>> list(typical['Section1'].keys()) - ['key'] - >>> list(typical['Section2'].keys()) - ['anotherkey'] - >>> custom = configparser.RawConfigParser() - >>> custom.optionxform = lambda option: option - >>> custom.read_string(config) - >>> list(custom['Section1'].keys()) - ['Key'] - >>> list(custom['Section2'].keys()) - ['AnotherKey'] - - .. note:: - The optionxform function transforms option names to a canonical form. - This should be an idempotent function: if the name is already in - canonical form, it should be returned unchanged. + .. note:: + The optionxform function transforms option names to a canonical form. + This should be an idempotent function: if the name is already in + canonical form, it should be returned unchanged. .. attribute:: ConfigParser.SECTCRE - A compiled regular expression used to parse section headers. The default - matches ``[section]`` to the name ``"section"``. Whitespace is considered - part of the section name, thus ``[ larch ]`` will be read as a section of - name ``" larch "``. Override this attribute if that's unsuitable. For - example: + A compiled regular expression used to parse section headers. The default + matches ``[section]`` to the name ``"section"``. Whitespace is considered + part of the section name, thus ``[ larch ]`` will be read as a section of + name ``" larch "``. Override this attribute if that's unsuitable. For + example: + + .. doctest:: + + >>> import re + >>> config = """ + ... [Section 1] + ... option = value + ... + ... [ Section 2 ] + ... another = val + ... """ + >>> typical = configparser.ConfigParser() + >>> typical.read_string(config) + >>> typical.sections() + ['Section 1', ' Section 2 '] + >>> custom = configparser.ConfigParser() + >>> custom.SECTCRE = re.compile(r"\[ *(?P
[^]]+?) *\]") + >>> custom.read_string(config) + >>> custom.sections() + ['Section 1', 'Section 2'] - .. doctest:: + .. note:: - >>> import re - >>> config = """ - ... [Section 1] - ... option = value - ... - ... [ Section 2 ] - ... another = val - ... """ - >>> typical = configparser.ConfigParser() - >>> typical.read_string(config) - >>> typical.sections() - ['Section 1', ' Section 2 '] - >>> custom = configparser.ConfigParser() - >>> custom.SECTCRE = re.compile(r"\[ *(?P
[^]]+?) *\]") - >>> custom.read_string(config) - >>> custom.sections() - ['Section 1', 'Section 2'] - - .. note:: - - While ConfigParser objects also use an ``OPTCRE`` attribute for recognizing - option lines, it's not recommended to override it because that would - interfere with constructor options *allow_no_value* and *delimiters*. + While ConfigParser objects also use an ``OPTCRE`` attribute for recognizing + option lines, it's not recommended to override it because that would + interfere with constructor options *allow_no_value* and *delimiters*. Legacy API Examples diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 501715980f9572..f17e1a37875168 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -31,7 +31,7 @@ A small number of constants live in the built-in namespace. They are: etc.) to indicate that the operation is not implemented with respect to the other type; may be returned by the in-place binary special methods (e.g. :meth:`__imul__`, :meth:`__iand__`, etc.) for the same purpose. - Its truth value is true. + It should not be evaluated in a boolean context. .. note:: @@ -50,6 +50,11 @@ A small number of constants live in the built-in namespace. They are: even though they have similar names and purposes. See :exc:`NotImplementedError` for details on when to use it. + .. versionchanged:: 3.9 + Evaluating ``NotImplemented`` in a boolean context is deprecated. While + it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. + It will raise a :exc:`TypeError` in a future version of Python. + .. index:: single: ...; ellipsis literal .. data:: Ellipsis diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 017a87a5648c39..0aa4ad76523480 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -416,7 +416,7 @@ Functions and classes provided: The passed in object is returned from the function, allowing this method to be used as a function decorator. - .. method:: callback(callback, *args, **kwds) + .. method:: callback(callback, /, *args, **kwds) Accepts an arbitrary callback function and arguments and adds it to the callback stack. @@ -473,7 +473,7 @@ Functions and classes provided: Similar to :meth:`push` but expects either an asynchronous context manager or a coroutine function. - .. method:: push_async_callback(callback, *args, **kwds) + .. method:: push_async_callback(callback, /, *args, **kwds) Similar to :meth:`callback` but expects a coroutine function. @@ -637,7 +637,7 @@ even further by means of a small helper class:: from contextlib import ExitStack class Callback(ExitStack): - def __init__(self, callback, *args, **kwds): + def __init__(self, callback, /, *args, **kwds): super(Callback, self).__init__() self.callback(callback, *args, **kwds) diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index c7bd89f9637226..176e01db6f9faf 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -31,6 +31,7 @@ Interface summary: Raised for module specific errors. +.. _shallow_vs_deep_copy: The difference between shallow and deep copying is only relevant for compound objects (objects that contain other objects, like lists or class instances): @@ -85,6 +86,7 @@ The latter is called to implement the deep copy operation; it is passed one argument, the ``memo`` dictionary. If the :meth:`__deepcopy__` implementation needs to make a deep copy of a component, it should call the :func:`deepcopy` function with the component as first argument and the memo dictionary as second argument. +The memo dictionary should be treated as an opaque object. .. seealso:: diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 40fca56d8029e9..43920210095951 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -49,7 +49,7 @@ The example below would like to show how to register a pickle function and how it will be used: >>> import copyreg, copy, pickle - >>> class C(object): + >>> class C: ... def __init__(self, a): ... self.a = a ... diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 17534fcc4615dc..7a72c26d5badeb 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -161,11 +161,15 @@ The :mod:`csv` module defines the following classes: If a row has more fields than fieldnames, the remaining data is put in a list and stored with the fieldname specified by *restkey* (which defaults to ``None``). If a non-blank row has fewer fields than fieldnames, the - missing values are filled-in with ``None``. + missing values are filled-in with the value of *restval* (which defaults + to ``None``). All other optional or keyword arguments are passed to the underlying :class:`reader` instance. + .. versionchanged:: 3.6 + Returned rows are now of type :class:`OrderedDict`. + .. versionchanged:: 3.8 Returned rows are now of type :class:`dict`. @@ -443,7 +447,8 @@ read CSV files (assuming they support complex numbers at all). .. method:: csvwriter.writerow(row) Write the *row* parameter to the writer's file object, formatted according to - the current dialect. + the current dialect. Return the return value of the call to the *write* method + of the underlying file object. .. versionchanged:: 3.5 Added support of arbitrary iterables. @@ -467,9 +472,14 @@ DictWriter objects have the following public method: .. method:: DictWriter.writeheader() - Write a row with the field names (as specified in the constructor). + Write a row with the field names (as specified in the constructor) to + the writer's file object, formatted according to the current dialect. Return + the return value of the :meth:`csvwriter.writerow` call used internally. .. versionadded:: 3.2 + .. versionchanged:: 3.8 + :meth:`writeheader` now also returns the value returned by + the :meth:`csvwriter.writerow` method it uses internally. .. _csv-examples: diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index baab0de8f8ac5e..2d6c6d0a1c3c57 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -152,8 +152,8 @@ the ``time()`` function, which returns system time in seconds since the Unix epoch, and the ``GetModuleHandleA()`` function, which returns a win32 module handle. -This example calls both functions with a NULL pointer (``None`` should be used -as the NULL pointer):: +This example calls both functions with a ``NULL`` pointer (``None`` should be used +as the ``NULL`` pointer):: >>> print(libc.time(None)) # doctest: +SKIP 1150640792 @@ -161,13 +161,6 @@ as the NULL pointer):: 0x1d000000 >>> -.. note:: - - :mod:`ctypes` may raise a :exc:`ValueError` after calling the function, if - it detects that an invalid number of arguments were passed. This behavior - should not be relied upon. It is deprecated in 3.6.2, and will be removed - in 3.7. - :exc:`ValueError` is raised when you call an ``stdcall`` function with the ``cdecl`` calling convention, or vice versa:: @@ -576,7 +569,7 @@ Here is a simple example of a POINT structure, which contains two integers named >>> POINT(1, 2, 3) Traceback (most recent call last): File "", line 1, in - ValueError: too many initializers + TypeError: too many initializers >>> You can, however, build much more complicated structures. A structure can @@ -624,7 +617,7 @@ Structure/union alignment and byte order ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, Structure and Union fields are aligned in the same way the C -compiler does it. It is possible to override this behavior be specifying a +compiler does it. It is possible to override this behavior by specifying a :attr:`_pack_` class attribute in the subclass definition. This must be set to a positive integer and specifies the maximum alignment for the fields. This is what ``#pragma pack(n)`` also does in MSVC. @@ -922,7 +915,7 @@ attribute later, after the class statement:: ... ("next", POINTER(cell))] >>> -Lets try it. We create two instances of ``cell``, and let them point to each +Let's try it. We create two instances of ``cell``, and let them point to each other, and finally follow the pointer chain a few times:: >>> c1 = cell() @@ -1083,7 +1076,7 @@ An extended example which also demonstrates the use of pointers accesses the Quoting the docs for that value: This pointer is initialized to point to an array of :c:type:`struct _frozen` - records, terminated by one whose members are all *NULL* or zero. When a frozen + records, terminated by one whose members are all ``NULL`` or zero. When a frozen module is imported, it is searched in this table. Third-party code could play tricks with this to provide a dynamically created collection of frozen modules. @@ -1110,7 +1103,7 @@ Since ``table`` is a ``pointer`` to the array of ``struct_frozen`` records, we can iterate over it, but we just have to make sure that our loop terminates, because pointers have no size. Sooner or later it would probably crash with an access violation or whatever, so it's better to break out of the loop when we -hit the NULL entry:: +hit the ``NULL`` entry:: >>> for item in table: ... if item.name is None: @@ -1125,8 +1118,8 @@ hit the NULL entry:: >>> The fact that standard Python has a frozen module and a frozen package -(indicated by the negative size member) is not well known, it is only used for -testing. Try it out with ``import __hello__`` for example. +(indicated by the negative ``size`` member) is not well known, it is only used +for testing. Try it out with ``import __hello__`` for example. .. _ctypes-surprises: @@ -1175,16 +1168,21 @@ Keep in mind that retrieving sub-objects from Structure, Unions, and Arrays doesn't *copy* the sub-object, instead it retrieves a wrapper object accessing the root-object's underlying buffer. -Another example that may behave different from what one would expect is this:: +Another example that may behave differently from what one would expect is this:: >>> s = c_char_p() - >>> s.value = "abc def ghi" + >>> s.value = b"abc def ghi" >>> s.value - 'abc def ghi' + b'abc def ghi' >>> s.value is s.value False >>> +.. note:: + + Objects instantiated from :class:`c_char_p` can only have their value set to bytes + or integers. + Why is it printing ``False``? ctypes instances are objects containing a memory block plus some :term:`descriptor`\s accessing the contents of the memory. Storing a Python object in the memory block does not store the object itself, @@ -1509,6 +1507,23 @@ object is available: :c:type:`int`, which is of course not always the truth, so you have to assign the correct :attr:`restype` attribute to use these functions. +.. audit-event:: ctypes.dlopen name ctypes.LibraryLoader + + Loading a library through any of these objects raises an + :ref:`auditing event ` ``ctypes.dlopen`` with string argument + ``name``, the name used to load the library. + +.. audit-event:: ctypes.dlsym library,name ctypes.LibraryLoader + + Accessing a function on a loaded library raises an auditing event + ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name`` + (the symbol's name as a string or integer). + +.. audit-event:: ctypes.dlsym/handle handle,name ctypes.LibraryLoader + + In cases when only the library handle is available rather than the object, + accessing a function raises an auditing event ``ctypes.dlsym/handle`` with + arguments ``handle`` (the raw library handle) and ``name``. .. _ctypes-foreign-functions: @@ -1595,6 +1610,19 @@ They are instances of a private class: passed arguments. +.. audit-event:: ctypes.seh_exception code foreign-functions + + On Windows, when a foreign function call raises a system exception (for + example, due to an access violation), it will be captured and replaced with + a suitable Python exception. Further, an auditing event + ``ctypes.seh_exception`` with argument ``code`` will be raised, allowing an + audit hook to replace the exception with its own. + +.. audit-event:: ctypes.call_function func_pointer,arguments ctype-foreign-functions + + Some ways to invoke foreign function calls may raise an auditing event + ``ctypes.call_function`` with arguments ``function pointer`` and ``arguments``. + .. _ctypes-function-prototypes: Function prototypes @@ -1786,6 +1814,8 @@ Utility functions Returns the address of the memory buffer as integer. *obj* must be an instance of a ctypes type. + .. audit-event:: ctypes.addressof obj ctypes.addressof + .. function:: alignment(obj_or_type) @@ -1828,6 +1858,7 @@ Utility functions termination character. An integer can be passed as second argument which allows specifying the size of the array if the length of the bytes should not be used. + .. audit-event:: ctypes.create_string_buffer init,size ctypes.create_string_buffer .. function:: create_unicode_buffer(init_or_size, size=None) @@ -1844,6 +1875,7 @@ Utility functions allows specifying the size of the array if the length of the string should not be used. + .. audit-event:: ctypes.create_unicode_buffer init,size ctypes.create_unicode_buffer .. function:: DllCanUnloadNow() @@ -1901,11 +1933,15 @@ Utility functions Returns the current value of the ctypes-private copy of the system :data:`errno` variable in the calling thread. + .. audit-event:: ctypes.get_errno "" ctypes.get_errno + .. function:: get_last_error() Windows only: returns the current value of the ctypes-private copy of the system :data:`LastError` variable in the calling thread. + .. audit-event:: ctypes.get_last_error "" ctypes.get_last_error + .. function:: memmove(dst, src, count) Same as the standard C memmove library function: copies *count* bytes from @@ -1949,6 +1985,7 @@ Utility functions Set the current value of the ctypes-private copy of the system :data:`errno` variable in the calling thread to *value* and return the previous value. + .. audit-event:: ctypes.set_errno errno ctypes.set_errno .. function:: set_last_error(value) @@ -1957,6 +1994,7 @@ Utility functions :data:`LastError` variable in the calling thread to *value* and return the previous value. + .. audit-event:: ctypes.set_last_error error ctypes.set_last_error .. function:: sizeof(obj_or_type) @@ -1971,6 +2009,8 @@ Utility functions object. If size is specified, it is used as size, otherwise the string is assumed to be zero-terminated. + .. audit-event:: ctypes.string_at address,size ctypes.string_at + .. function:: WinError(code=None, descr=None) @@ -1991,6 +2031,8 @@ Utility functions characters of the string, otherwise the string is assumed to be zero-terminated. + .. audit-event:: ctypes.wstring_at address,size ctypes.wstring_at + .. _ctypes-data-types: @@ -2018,6 +2060,7 @@ Data types source buffer in bytes; the default is zero. If the source buffer is not large enough a :exc:`ValueError` is raised. + .. audit-event:: ctypes.cdata/buffer pointer,size,offset ctypes._CData.from_buffer .. method:: _CData.from_buffer_copy(source[, offset]) @@ -2027,11 +2070,19 @@ Data types is zero. If the source buffer is not large enough a :exc:`ValueError` is raised. + .. audit-event:: ctypes.cdata/buffer pointer,size,offset ctypes._CData.from_buffer_copy + .. method:: from_address(address) This method returns a ctypes type instance using the memory specified by *address* which must be an integer. + .. audit-event:: ctypes.cdata address ctypes._CData.from_address + + This method, and others that indirectly call this method, raises an + :ref:`auditing event ` ``ctypes.cdata`` with argument + ``address``. + .. method:: from_param(obj) This method adapts *obj* to a ctypes type. It is called with the actual @@ -2376,7 +2427,7 @@ other data types containing pointer type fields. and so on). Later assignments to the :attr:`_fields_` class variable will raise an AttributeError. - It is possible to defined sub-subclasses of structure types, they inherit + It is possible to define sub-subclasses of structure types, they inherit the fields of the base class plus the :attr:`_fields_` defined in the sub-subclass, if any. @@ -2424,7 +2475,7 @@ other data types containing pointer type fields. td.lptdesc = POINTER(some_type) td.u.lptdesc = POINTER(some_type) - It is possible to defined sub-subclasses of structures, they inherit the + It is possible to define sub-subclasses of structures, they inherit the fields of the base class. If the subclass definition has a separate :attr:`_fields_` variable, the fields specified in this are appended to the fields of the base class. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 2a4d9ce8a35a4e..3684d54d4ecade 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -214,7 +214,7 @@ The module :mod:`curses` defines the following functions: .. function:: getmouse() After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this - method should be call to retrieve the queued mouse event, represented as a + method should be called to retrieve the queued mouse event, represented as a 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is currently unused.) *bstate* is an integer value whose bits will be set to @@ -242,6 +242,15 @@ The module :mod:`curses` defines the following functions: Return ``True`` if the terminal can display colors; otherwise, return ``False``. +.. function:: has_extended_color_support() + + Return ``True`` if the module supports extended colors; otherwise, return + ``False``. Extended color support allows more than 256 color pairs for + terminals that support more than 16 colors (e.g. xterm-256color). + + Extended color support requires ncurses version 6.1 or later. + + .. versionadded:: 3.10 .. function:: has_ic() @@ -511,6 +520,32 @@ The module :mod:`curses` defines the following functions: Save the current state of the terminal modes in a buffer, usable by :func:`resetty`. +.. function:: get_escdelay() + + Retrieves the value set by :func:`set_escdelay`. + + .. versionadded:: 3.9 + +.. function:: set_escdelay(ms) + + Sets the number of milliseconds to wait after reading an escape character, + to distinguish between an individual escape character entered on the + keyboard from escape sequences sent by cursor and function keys. + + .. versionadded:: 3.9 + +.. function:: get_tabsize() + + Retrieves the value set by :func:`set_tabsize`. + + .. versionadded:: 3.9 + +.. function:: set_tabsize(size) + + Sets the number of columns used by the curses library when converting a tab + character to spaces as it adds the tab to a window. + + .. versionadded:: 3.9 .. function:: setsyx(y, x) @@ -656,7 +691,7 @@ The module :mod:`curses` defines the following functions: foreground color on the default background. -.. function:: wrapper(func, ...) +.. function:: wrapper(func, /, *args, **kwargs) Initialize curses and call another callable object, *func*, which should be the rest of your curses-using application. If the application raises an exception, @@ -682,7 +717,7 @@ the following methods and attributes: window.addch(y, x, ch[, attr]) Paint character *ch* at ``(y, x)`` with attributes *attr*, overwriting any - character previously painter at that location. By default, the character + character previously painted at that location. By default, the character position and attributes are the current settings for the window object. .. note:: @@ -708,9 +743,16 @@ the following methods and attributes: .. note:: - Writing outside the window, subwindow, or pad raises :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, - or pad will cause an exception to be raised after the string is printed. + * Writing outside the window, subwindow, or pad raises :exc:`curses.error`. + Attempting to write to the lower right corner of a window, subwindow, + or pad will cause an exception to be raised after the string is printed. + + * A `bug in ncurses `_, the backend + for this Python module, can cause SegFaults when resizing windows. This + is fixed in ncurses-6.1-20190511. If you are stuck with an earlier + ncurses, you can avoid triggering this if you do not call :func:`addstr` + with a *str* that has embedded newlines. Instead, call :func:`addstr` + separately for each line. .. method:: window.attroff(attr) @@ -1276,7 +1318,7 @@ The :mod:`curses` module defines the following data members: .. data:: ERR - Some curses routines that return an integer, such as :func:`getch`, return + Some curses routines that return an integer, such as :meth:`~window.getch`, return :const:`ERR` upon failure. diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index db5c3e0c7e2893..6e74af062d9e72 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -19,9 +19,11 @@ in :pep:`557`. The member variables to use in these generated methods are defined using :pep:`526` type annotations. For example this code:: + from dataclasses import dataclass + @dataclass class InventoryItem: - '''Class for keeping track of an item in inventory.''' + """Class for keeping track of an item in inventory.""" name: str unit_price: float quantity_on_hand: int = 0 @@ -60,8 +62,9 @@ Module-level decorators, classes, and functions The :func:`dataclass` decorator will add various "dunder" methods to the class, described below. If any of the added methods already - exist on the class, a :exc:`TypeError` will be raised. The decorator - returns the same class that is called on: no new class is created. + exist on the class, the behavior depends on the parameter, as documented + below. The decorator returns the same class that is called on; no new + class is created. If :func:`dataclass` is used just as a simple decorator with no parameters, it acts as if it has the default values documented in this @@ -115,7 +118,7 @@ Module-level decorators, classes, and functions If the class already defines any of :meth:`__lt__`, :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then - :exc:`ValueError` is raised. + :exc:`TypeError` is raised. - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method is generated according to how ``eq`` and ``frozen`` are set. @@ -153,7 +156,7 @@ Module-level decorators, classes, and functions method of the superclass will be used (if the superclass is :class:`object`, this means it will fall back to id-based hashing). - - ``frozen``: If true (the default is False), assigning to fields will + - ``frozen``: If true (the default is ``False``), assigning to fields will generate an exception. This emulates read-only frozen instances. If :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then :exc:`TypeError` is raised. See the discussion below. @@ -356,7 +359,7 @@ Module-level decorators, classes, and functions def add_one(self): return self.x + 1 -.. function:: replace(instance, **changes) +.. function:: replace(instance, /, **changes) Creates a new object of the same type of ``instance``, replacing fields with values from ``changes``. If ``instance`` is not a Data @@ -386,8 +389,8 @@ Module-level decorators, classes, and functions .. function:: is_dataclass(class_or_instance) - Returns True if its parameter is a dataclass or an instance of one, - otherwise returns False. + Return ``True`` if its parameter is a dataclass or an instance of one, + otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst index 48af0823aa4912..ff51b2779e5fa3 100644 --- a/Doc/library/datatypes.rst +++ b/Doc/library/datatypes.rst @@ -5,13 +5,14 @@ Data Types ********** The modules described in this chapter provide a variety of specialized data -types such as dates and times, fixed-type arrays, heap queues, synchronized -queues, and sets. +types such as dates and times, fixed-type arrays, heap queues, double-ended +queues, and enumerations. Python also provides some built-in data types, in particular, :class:`dict`, :class:`list`, :class:`set` and :class:`frozenset`, and :class:`tuple`. The :class:`str` class is used to hold -Unicode strings, and the :class:`bytes` class is used to hold binary data. +Unicode strings, and the :class:`bytes` and :class:`bytearray` classes are used +to hold binary data. The following modules are documented in this chapter: @@ -19,6 +20,7 @@ The following modules are documented in this chapter: .. toctree:: datetime.rst + zoneinfo.rst calendar.rst collections.rst collections.abc.rst @@ -31,3 +33,4 @@ The following modules are documented in this chapter: pprint.rst reprlib.rst enum.rst + graphlib.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 1ee23c2175a27d..508bc88e7f4b8a 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -14,40 +14,60 @@ .. XXX what order should the types be discussed in? -The :mod:`datetime` module supplies classes for manipulating dates and times in -both simple and complex ways. While date and time arithmetic is supported, the -focus of the implementation is on efficient attribute extraction for output -formatting and manipulation. For related functionality, see also the -:mod:`time` and :mod:`calendar` modules. - -There are two kinds of date and time objects: "naive" and "aware". - -An aware object has sufficient knowledge of applicable algorithmic and -political time adjustments, such as time zone and daylight saving time -information, to locate itself relative to other aware objects. An aware object -is used to represent a specific moment in time that is not open to -interpretation [#]_. - -A naive object does not contain enough information to unambiguously locate -itself relative to other date/time objects. Whether a naive object represents +The :mod:`datetime` module supplies classes for manipulating dates and times. + +While date and time arithmetic is supported, the focus of the implementation is +on efficient attribute extraction for output formatting and manipulation. + +.. seealso:: + + Module :mod:`calendar` + General calendar related functions. + + Module :mod:`time` + Time access and conversions. + + Package `dateutil `_ + Third-party library with expanded time zone and parsing support. + +.. _datetime-naive-aware: + +Aware and Naive Objects +----------------------- + +Date and time objects may be categorized as "aware" or "naive" depending on +whether or not they include timezone information. + +With sufficient knowledge of applicable algorithmic and political time +adjustments, such as time zone and daylight saving time information, +an **aware** object can locate itself relative to other aware objects. +An aware object represents a specific moment in time that is not open to +interpretation. [#]_ + +A **naive** object does not contain enough information to unambiguously locate +itself relative to other date/time objects. Whether a naive object represents Coordinated Universal Time (UTC), local time, or time in some other timezone is purely up to the program, just like it is up to the program whether a -particular number represents metres, miles, or mass. Naive objects are easy to +particular number represents metres, miles, or mass. Naive objects are easy to understand and to work with, at the cost of ignoring some aspects of reality. For applications requiring aware objects, :class:`.datetime` and :class:`.time` objects have an optional time zone information attribute, :attr:`!tzinfo`, that can be set to an instance of a subclass of the abstract :class:`tzinfo` class. These :class:`tzinfo` objects capture information about the offset from UTC -time, the time zone name, and whether Daylight Saving Time is in effect. Note -that only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`datetime` module. The :class:`timezone` class can -represent simple timezones with fixed offset from UTC, such as UTC itself or -North American EST and EDT timezones. Supporting timezones at deeper levels of -detail is up to the application. The rules for time adjustment across the +time, the time zone name, and whether daylight saving time is in effect. + +Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is +supplied by the :mod:`datetime` module. The :class:`timezone` class can +represent simple timezones with fixed offsets from UTC, such as UTC itself or +North American EST and EDT timezones. Supporting timezones at deeper levels of +detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. +Constants +--------- + The :mod:`datetime` module exports the following constants: .. data:: MINYEAR @@ -61,16 +81,6 @@ The :mod:`datetime` module exports the following constants: The largest year number allowed in a :class:`date` or :class:`.datetime` object. :const:`MAXYEAR` is ``9999``. - -.. seealso:: - - Module :mod:`calendar` - General calendar related functions. - - Module :mod:`time` - Time access and conversions. - - Available Types --------------- @@ -86,7 +96,7 @@ Available Types :noindex: An idealized time, independent of any particular day, assuming that every day - has exactly 24\*60\*60 seconds (there is no notion of "leap seconds" here). + has exactly 24\*60\*60 seconds. (There is no notion of "leap seconds" here.) Attributes: :attr:`hour`, :attr:`minute`, :attr:`second`, :attr:`microsecond`, and :attr:`.tzinfo`. @@ -109,7 +119,7 @@ Available Types .. class:: tzinfo :noindex: - An abstract base class for time zone information objects. These are used by the + An abstract base class for time zone information objects. These are used by the :class:`.datetime` and :class:`.time` classes to provide a customizable notion of time adjustment (for example, to account for time zone and/or daylight saving time). @@ -122,22 +132,8 @@ Available Types .. versionadded:: 3.2 - Objects of these types are immutable. -Objects of the :class:`date` type are always naive. - -An object of type :class:`.time` or :class:`.datetime` may be naive or aware. -A :class:`.datetime` object *d* is aware if ``d.tzinfo`` is not ``None`` and -``d.tzinfo.utcoffset(d)`` does not return ``None``. If ``d.tzinfo`` is -``None``, or if ``d.tzinfo`` is not ``None`` but ``d.tzinfo.utcoffset(d)`` -returns ``None``, *d* is naive. A :class:`.time` object *t* is aware -if ``t.tzinfo`` is not ``None`` and ``t.tzinfo.utcoffset(None)`` does not return -``None``. Otherwise, *t* is naive. - -The distinction between naive and aware doesn't apply to :class:`timedelta` -objects. - Subclass relationships:: object @@ -148,6 +144,40 @@ Subclass relationships:: date datetime +Common Properties +^^^^^^^^^^^^^^^^^ + +The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types +share these common features: + +- Objects of these types are immutable. +- Objects of these types are hashable, meaning that they can be used as + dictionary keys. +- Objects of these types support efficient pickling via the :mod:`pickle` module. + +Determining if an Object is Aware or Naive +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Objects of the :class:`date` type are always naive. + +An object of type :class:`.time` or :class:`.datetime` may be aware or naive. + +A :class:`.datetime` object *d* is aware if both of the following hold: + +1. ``d.tzinfo`` is not ``None`` +2. ``d.tzinfo.utcoffset(d)`` does not return ``None`` + +Otherwise, *d* is naive. + +A :class:`.time` object *t* is aware if both of the following hold: + +1. ``t.tzinfo`` is not ``None`` +2. ``t.tzinfo.utcoffset(None)`` does not return ``None``. + +Otherwise, *t* is naive. + +The distinction between aware and naive doesn't apply to :class:`timedelta` +objects. .. _datetime-timedelta: @@ -159,11 +189,11 @@ dates or times. .. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) - All arguments are optional and default to ``0``. Arguments may be integers + All arguments are optional and default to ``0``. Arguments may be integers or floats, and may be positive or negative. - Only *days*, *seconds* and *microseconds* are stored internally. Arguments are - converted to those units: + Only *days*, *seconds* and *microseconds* are stored internally. + Arguments are converted to those units: * A millisecond is converted to 1000 microseconds. * A minute is converted to 60 seconds. @@ -177,10 +207,28 @@ dates or times. * ``0 <= seconds < 3600*24`` (the number of seconds in one day) * ``-999999999 <= days <= 999999999`` + The following example illustrates how any arguments besides + *days*, *seconds* and *microseconds* are "merged" and normalized into those + three resulting attributes:: + + >>> from datetime import timedelta + >>> delta = timedelta( + ... days=50, + ... seconds=27, + ... microseconds=10, + ... milliseconds=29000, + ... minutes=5, + ... hours=8, + ... weeks=2 + ... ) + >>> # Only days, seconds, and microseconds remain + >>> delta + datetime.timedelta(days=64, seconds=29156, microseconds=10) + If any argument is a float and there are fractional microseconds, the fractional microseconds left over from all arguments are combined and their sum is rounded to the nearest microsecond using - round-half-to-even tiebreaker. If no argument is a float, the + round-half-to-even tiebreaker. If no argument is a float, the conversion and normalization processes are exact (no information is lost). @@ -188,7 +236,7 @@ dates or times. :exc:`OverflowError` is raised. Note that normalization of negative values may be surprising at first. For - example, + example:: >>> from datetime import timedelta >>> d = timedelta(microseconds=-1) @@ -196,7 +244,7 @@ dates or times. (-1, 86399, 999999) -Class attributes are: +Class attributes: .. attribute:: timedelta.min @@ -263,7 +311,7 @@ Supported operations: | | timedelta.resolution using round-half-to-even.| +--------------------------------+-----------------------------------------------+ | ``t1 = t2 // i`` or | The floor is computed and the remainder (if | -| ``t1 = t2 // t3`` | any) is thrown away. In the second case, an | +| ``t1 = t2 // t3`` | any) is thrown away. In the second case, an | | | integer is returned. (3) | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 % t3`` | The remainder is computed as a | @@ -282,8 +330,8 @@ Supported operations: | | -*t1.seconds*, -*t1.microseconds*), | | | and to *t1*\* -1. (1)(4) | +--------------------------------+-----------------------------------------------+ -| ``abs(t)`` | equivalent to +\ *t* when ``t.days >= 0``, and| -| | to -*t* when ``t.days < 0``. (2) | +| ``abs(t)`` | equivalent to +\ *t* when ``t.days >= 0``, | +| | and to -*t* when ``t.days < 0``. (2) | +--------------------------------+-----------------------------------------------+ | ``str(t)`` | Returns a string in the form | | | ``[D day[s], ][H]H:MM:SS[.UUUUUU]``, where D | @@ -298,10 +346,10 @@ Supported operations: Notes: (1) - This is exact, but may overflow. + This is exact but may overflow. (2) - This is exact, and cannot overflow. + This is exact and cannot overflow. (3) Division by 0 raises :exc:`ZeroDivisionError`. @@ -310,41 +358,56 @@ Notes: -*timedelta.max* is not representable as a :class:`timedelta` object. (5) - String representations of :class:`timedelta` objects are normalized - similarly to their internal representation. This leads to somewhat - unusual results for negative timedeltas. For example: + String representations of :class:`timedelta` objects are normalized + similarly to their internal representation. This leads to somewhat + unusual results for negative timedeltas. For example:: - >>> timedelta(hours=-5) - datetime.timedelta(days=-1, seconds=68400) - >>> print(_) - -1 day, 19:00:00 + >>> timedelta(hours=-5) + datetime.timedelta(days=-1, seconds=68400) + >>> print(_) + -1 day, 19:00:00 (6) The expression ``t2 - t3`` will always be equal to the expression ``t2 + (-t3)`` except when t3 is equal to ``timedelta.max``; in that case the former will produce a result while the latter will overflow. -In addition to the operations listed above :class:`timedelta` objects support +In addition to the operations listed above, :class:`timedelta` objects support certain additions and subtractions with :class:`date` and :class:`.datetime` objects (see below). .. versionchanged:: 3.2 Floor division and true division of a :class:`timedelta` object by another :class:`timedelta` object are now supported, as are remainder operations and - the :func:`divmod` function. True division and multiplication of a + the :func:`divmod` function. True division and multiplication of a :class:`timedelta` object by a :class:`float` object are now supported. -Comparisons of :class:`timedelta` objects are supported with the -:class:`timedelta` object representing the smaller duration considered to be the -smaller timedelta. In order to stop mixed-type comparisons from falling back to -the default comparison by object address, when a :class:`timedelta` object is -compared to an object of a different type, :exc:`TypeError` is raised unless the -comparison is ``==`` or ``!=``. The latter cases return :const:`False` or -:const:`True`, respectively. +Comparisons of :class:`timedelta` objects are supported, with some caveats. + +The comparisons ``==`` or ``!=`` *always* return a :class:`bool`, no matter +the type of the compared object:: + + >>> from datetime import timedelta + >>> delta1 = timedelta(seconds=57) + >>> delta2 = timedelta(hours=25, seconds=2) + >>> delta2 != delta1 + True + >>> delta2 == 5 + False -:class:`timedelta` objects are :term:`hashable` (usable as dictionary keys), support -efficient pickling, and in Boolean contexts, a :class:`timedelta` object is +For all other comparisons (such as ``<`` and ``>``), when a :class:`timedelta` +object is compared to an object of a different type, :exc:`TypeError` +is raised:: + + >>> delta2 > delta1 + True + >>> delta2 > 5 + Traceback (most recent call last): + File "", line 1, in + TypeError: '>' not supported between instances of 'datetime.timedelta' and 'int' + +In Boolean contexts, a :class:`timedelta` object is considered to be true if and only if it isn't equal to ``timedelta(0)``. Instance methods: @@ -360,29 +423,36 @@ Instance methods: .. versionadded:: 3.2 +Examples of usage: :class:`timedelta` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Example usage: +An additional example of normalization:: + >>> # Components of another_year add up to exactly 365 days >>> from datetime import timedelta >>> year = timedelta(days=365) >>> another_year = timedelta(weeks=40, days=84, hours=23, - ... minutes=50, seconds=600) # adds up to 365 days - >>> year.total_seconds() - 31536000.0 + ... minutes=50, seconds=600) >>> year == another_year True + >>> year.total_seconds() + 31536000.0 + +Examples of :class:`timedelta` arithmetic:: + + >>> from datetime import timedelta + >>> year = timedelta(days=365) >>> ten_years = 10 * year - >>> ten_years, ten_years.days // 365 - (datetime.timedelta(days=3650), 10) + >>> ten_years + datetime.timedelta(days=3650) + >>> ten_years.days // 365 + 10 >>> nine_years = ten_years - year - >>> nine_years, nine_years.days // 365 - (datetime.timedelta(days=3285), 9) + >>> nine_years + datetime.timedelta(days=3285) >>> three_years = nine_years // 3 >>> three_years, three_years.days // 365 (datetime.timedelta(days=1095), 3) - >>> abs(three_years - ten_years) == 2 * three_years + year - True - .. _datetime-date: @@ -391,17 +461,14 @@ Example usage: A :class:`date` object represents a date (year, month and day) in an idealized calendar, the current Gregorian calendar indefinitely extended in both -directions. January 1 of year 1 is called day number 1, January 2 of year 1 is -called day number 2, and so on. This matches the definition of the "proleptic -Gregorian" calendar in Dershowitz and Reingold's book Calendrical Calculations, -where it's the base calendar for all computations. See the book for algorithms -for converting between proleptic Gregorian ordinals and many other calendar -systems. +directions. +January 1 of year 1 is called day number 1, January 2 of year 1 is +called day number 2, and so on. [#]_ .. class:: date(year, month, day) - All arguments are required. Arguments may be integers, in the following + All arguments are required. Arguments must be integers, in the following ranges: * ``MINYEAR <= year <= MAXYEAR`` @@ -415,17 +482,19 @@ Other constructors, all class methods: .. classmethod:: date.today() - Return the current local date. This is equivalent to - ``date.fromtimestamp(time.time())``. + Return the current local date. + This is equivalent to ``date.fromtimestamp(time.time())``. .. classmethod:: date.fromtimestamp(timestamp) - Return the local date corresponding to the POSIX timestamp, such as is returned - by :func:`time.time`. This may raise :exc:`OverflowError`, if the timestamp is out - of the range of values supported by the platform C :c:func:`localtime` function, - and :exc:`OSError` on :c:func:`localtime` failure. - It's common for this to be restricted to years from 1970 through 2038. Note + Return the local date corresponding to the POSIX timestamp, such as is + returned by :func:`time.time`. + + This may raise :exc:`OverflowError`, if the timestamp is out + of the range of values supported by the platform C :c:func:`localtime` + function, and :exc:`OSError` on :c:func:`localtime` failure. + It's common for this to be restricted to years from 1970 through 2038. Note that on non-POSIX systems that include leap seconds in their notion of a timestamp, leap seconds are ignored by :meth:`fromtimestamp`. @@ -438,25 +507,35 @@ Other constructors, all class methods: .. classmethod:: date.fromordinal(ordinal) - Return the date corresponding to the proleptic Gregorian ordinal, where January - 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= - date.max.toordinal()``. For any date *d*, ``date.fromordinal(d.toordinal()) == - d``. + Return the date corresponding to the proleptic Gregorian ordinal, where + January 1 of year 1 has ordinal 1. + + :exc:`ValueError` is raised unless ``1 <= ordinal <= + date.max.toordinal()``. For any date *d*, + ``date.fromordinal(d.toordinal()) == d``. .. classmethod:: date.fromisoformat(date_string) - Return a :class:`date` corresponding to a *date_string* in the format emitted - by :meth:`date.isoformat`. Specifically, this function supports strings in - the format(s) ``YYYY-MM-DD``. + Return a :class:`date` corresponding to a *date_string* given in the format + ``YYYY-MM-DD``:: + + >>> from datetime import date + >>> date.fromisoformat('2019-12-04') + datetime.date(2019, 12, 4) + + This is the inverse of :meth:`date.isoformat`. It only supports the format + ``YYYY-MM-DD``. + + .. versionadded:: 3.7 - .. caution:: - This does not support parsing arbitrary ISO 8601 strings - it is only intended - as the inverse operation of :meth:`date.isoformat`. +.. classmethod:: date.fromisocalendar(year, week, day) - .. versionadded:: 3.7 + Return a :class:`date` corresponding to the ISO calendar date specified by + year, week and day. This is the inverse of the function :meth:`date.isocalendar`. + .. versionadded:: 3.8 Class attributes: @@ -500,7 +579,7 @@ Supported operations: | Operation | Result | +===============================+==============================================+ | ``date2 = date1 + timedelta`` | *date2* is ``timedelta.days`` days removed | -| | from *date1*. (1) | +| | from *date1*. (1) | +-------------------------------+----------------------------------------------+ | ``date2 = date1 - timedelta`` | Computes *date2* such that ``date2 + | | | timedelta == date1``. (2) | @@ -515,7 +594,7 @@ Notes: (1) *date2* is moved forward in time if ``timedelta.days > 0``, or backward if - ``timedelta.days < 0``. Afterward ``date2 - date1 == timedelta.days``. + ``timedelta.days < 0``. Afterward ``date2 - date1 == timedelta.days``. ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. :exc:`OverflowError` is raised if ``date2.year`` would be smaller than :const:`MINYEAR` or larger than :const:`MAXYEAR`. @@ -524,7 +603,7 @@ Notes: ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. (3) - This is exact, and cannot overflow. timedelta.seconds and + This is exact, and cannot overflow. timedelta.seconds and timedelta.microseconds are 0, and date2 + timedelta == date1 after. (4) @@ -532,38 +611,47 @@ Notes: date2.toordinal()``. Date comparison raises :exc:`TypeError` if the other comparand isn't also a :class:`date` object. However, ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a + :meth:`timetuple` attribute. This hook gives other kinds of date objects a chance at implementing mixed-type comparison. If not, when a :class:`date` object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return + unless the comparison is ``==`` or ``!=``. The latter cases return :const:`False` or :const:`True`, respectively. -Dates can be used as dictionary keys. In Boolean contexts, all :class:`date` -objects are considered to be true. +In Boolean contexts, all :class:`date` objects are considered to be true. Instance methods: .. method:: date.replace(year=self.year, month=self.month, day=self.day) Return a date with the same value, except for those parameters given new - values by whichever keyword arguments are specified. For example, if ``d == - date(2002, 12, 31)``, then ``d.replace(day=26) == date(2002, 12, 26)``. + values by whichever keyword arguments are specified. + + Example:: + + >>> from datetime import date + >>> d = date(2002, 12, 31) + >>> d.replace(day=26) + datetime.date(2002, 12, 26) .. method:: date.timetuple() Return a :class:`time.struct_time` such as returned by :func:`time.localtime`. - The hours, minutes and seconds are 0, and the DST flag is -1. ``d.timetuple()`` - is equivalent to ``time.struct_time((d.year, d.month, d.day, 0, 0, 0, - d.weekday(), yday, -1))``, where ``yday = d.toordinal() - date(d.year, 1, - 1).toordinal() + 1`` is the day number within the current year starting with - ``1`` for January 1st. + + The hours, minutes and seconds are 0, and the DST flag is -1. + + ``d.timetuple()`` is equivalent to:: + + time.struct_time((d.year, d.month, d.day, 0, 0, 0, d.weekday(), yday, -1)) + + where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` + is the day number within the current year starting with ``1`` for January 1st. .. method:: date.toordinal() Return the proleptic Gregorian ordinal of the date, where January 1 of year 1 - has ordinal 1. For any :class:`date` object *d*, + has ordinal 1. For any :class:`date` object *d*, ``date.fromordinal(d.toordinal()) == d``. @@ -583,28 +671,37 @@ Instance methods: .. method:: date.isocalendar() - Return a 3-tuple, (ISO year, ISO week number, ISO weekday). + Return a :term:`named tuple` object with three components: ``year``, + ``week`` and ``weekday``. - The ISO calendar is a widely used variant of the Gregorian calendar. See - https://www.staff.science.uu.nl/~gent0113/calendar/isocalendar.htm for a good - explanation. + The ISO calendar is a widely used variant of the Gregorian calendar. [#]_ The ISO year consists of 52 or 53 full weeks, and where a week starts on a - Monday and ends on a Sunday. The first week of an ISO year is the first + Monday and ends on a Sunday. The first week of an ISO year is the first (Gregorian) calendar week of a year containing a Thursday. This is called week number 1, and the ISO year of that Thursday is the same as its Gregorian year. For example, 2004 begins on a Thursday, so the first week of ISO year 2004 - begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004, so that - ``date(2003, 12, 29).isocalendar() == (2004, 1, 1)`` and ``date(2004, 1, - 4).isocalendar() == (2004, 1, 7)``. + begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004:: + + >>> from datetime import date + >>> date(2003, 12, 29).isocalendar() + datetime.IsoCalendarDate(year=2004, week=1, weekday=1) + >>> date(2004, 1, 4).isocalendar() + datetime.IsoCalendarDate(year=2004, week=1, weekday=7) + .. versionchanged:: 3.9 + Result changed from a tuple to a :term:`named tuple`. .. method:: date.isoformat() - Return a string representing the date in ISO 8601 format, 'YYYY-MM-DD'. For - example, ``date(2002, 12, 4).isoformat() == '2002-12-04'``. + Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``:: + >>> from datetime import date + >>> date(2002, 12, 4).isoformat() + '2002-12-04' + + This is the inverse of :meth:`date.fromisoformat`. .. method:: date.__str__() @@ -613,9 +710,17 @@ Instance methods: .. method:: date.ctime() - Return a string representing the date, for example ``date(2002, 12, - 4).ctime() == 'Wed Dec 4 00:00:00 2002'``. ``d.ctime()`` is equivalent to - ``time.ctime(time.mktime(d.timetuple()))`` on platforms where the native C + Return a string representing the date:: + + >>> from datetime import date + >>> date(2002, 12, 4).ctime() + 'Wed Dec 4 00:00:00 2002' + + ``d.ctime()`` is equivalent to:: + + time.ctime(time.mktime(d.timetuple())) + + on platforms where the native C :c:func:`ctime` function (which :func:`time.ctime` invokes, but which :meth:`date.ctime` does not invoke) conforms to the C standard. @@ -636,6 +741,8 @@ Instance methods: complete list of formatting directives, see :ref:`strftime-strptime-behavior`. +Examples of Usage: :class:`date` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example of counting days to an event:: @@ -655,7 +762,7 @@ Example of counting days to an event:: >>> time_to_birthday.days 202 -Example of working with :class:`date`: +More examples of working with :class:`date`: .. doctest:: @@ -663,6 +770,20 @@ Example of working with :class:`date`: >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 >>> d datetime.date(2002, 3, 11) + + >>> # Methods related to formatting string output + >>> d.isoformat() + '2002-03-11' + >>> d.strftime("%d/%m/%y") + '11/03/02' + >>> d.strftime("%A %d. %B %Y") + 'Monday 11. March 2002' + >>> d.ctime() + 'Mon Mar 11 00:00:00 2002' + >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") + 'The day is 11, the month is March.' + + >>> # Methods for to extracting 'components' under different calendars >>> t = d.timetuple() >>> for i in t: # doctest: +SKIP ... print(i) @@ -681,14 +802,10 @@ Example of working with :class:`date`: 2002 # ISO year 11 # ISO week number 1 # ISO day number ( 1 = Monday ) - >>> d.isoformat() - '2002-03-11' - >>> d.strftime("%d/%m/%y") - '11/03/02' - >>> d.strftime("%A %d. %B %Y") - 'Monday 11. March 2002' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") - 'The day is 11, the month is March.' + + >>> # A date object is immutable; all operations produce a new object + >>> d.replace(year=2005) + datetime.date(2005, 3, 11) .. _datetime-datetime: @@ -697,17 +814,18 @@ Example of working with :class:`date`: -------------------------- A :class:`.datetime` object is a single object containing all the information -from a :class:`date` object and a :class:`.time` object. Like a :class:`date` -object, :class:`.datetime` assumes the current Gregorian calendar extended in -both directions; like a time object, :class:`.datetime` assumes there are exactly -3600\*24 seconds in every day. +from a :class:`date` object and a :class:`.time` object. + +Like a :class:`date` object, :class:`.datetime` assumes the current Gregorian +calendar extended in both directions; like a :class:`.time` object, +:class:`.datetime` assumes there are exactly 3600\*24 seconds in every day. Constructor: .. class:: datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) - The year, month and day arguments are required. *tzinfo* may be ``None``, or an - instance of a :class:`tzinfo` subclass. The remaining arguments may be integers, + The *year*, *month* and *day* arguments are required. *tzinfo* may be ``None``, or an + instance of a :class:`tzinfo` subclass. The remaining arguments must be integers in the following ranges: * ``MINYEAR <= year <= MAXYEAR``, @@ -728,31 +846,48 @@ Other constructors, all class methods: .. classmethod:: datetime.today() - Return the current local datetime, with :attr:`.tzinfo` ``None``. This is - equivalent to ``datetime.fromtimestamp(time.time())``. See also :meth:`now`, - :meth:`fromtimestamp`. + Return the current local datetime, with :attr:`.tzinfo` ``None``. + + Equivalent to:: + datetime.fromtimestamp(time.time()) + + See also :meth:`now`, :meth:`fromtimestamp`. + + This method is functionally equivalent to :meth:`now`, but without a + ``tz`` parameter. .. classmethod:: datetime.now(tz=None) - Return the current local date and time. If optional argument *tz* is ``None`` + Return the current local date and time. + + If optional argument *tz* is ``None`` or not specified, this is like :meth:`today`, but, if possible, supplies more precision than can be gotten from going through a :func:`time.time` timestamp (for example, this may be possible on platforms supplying the C :c:func:`gettimeofday` function). - If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the - current date and time are converted to *tz*’s time zone. In this case the - result is equivalent to ``tz.fromutc(datetime.utcnow().replace(tzinfo=tz))``. - See also :meth:`today`, :meth:`utcnow`. + If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, + and the current date and time are converted to *tz*’s time zone. + + This function is preferred over :meth:`today` and :meth:`utcnow`. .. classmethod:: datetime.utcnow() - Return the current UTC date and time, with :attr:`.tzinfo` ``None``. This is like - :meth:`now`, but returns the current UTC date and time, as a naive - :class:`.datetime` object. An aware current UTC datetime can be obtained by - calling ``datetime.now(timezone.utc)``. See also :meth:`now`. + Return the current UTC date and time, with :attr:`.tzinfo` ``None``. + + This is like :meth:`now`, but returns the current UTC date and time, as a naive + :class:`.datetime` object. An aware current UTC datetime can be obtained by + calling ``datetime.now(timezone.utc)``. See also :meth:`now`. + + .. warning:: + + Because naive ``datetime`` objects are treated by many ``datetime`` methods + as local times, it is preferred to use aware datetimes to represent times + in UTC. As such, the recommended way to create an object representing the + current time in UTC is by calling ``datetime.now(timezone.utc)``. + .. classmethod:: datetime.fromtimestamp(timestamp, tz=None) @@ -762,9 +897,7 @@ Other constructors, all class methods: the returned :class:`.datetime` object is naive. If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the - timestamp is converted to *tz*’s time zone. In this case the result is - equivalent to - ``tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))``. + timestamp is converted to *tz*’s time zone. :meth:`fromtimestamp` may raise :exc:`OverflowError`, if the timestamp is out of the range of values supported by the platform C :c:func:`localtime` or @@ -774,7 +907,8 @@ Other constructors, all class methods: 1970 through 2038. Note that on non-POSIX systems that include leap seconds in their notion of a timestamp, leap seconds are ignored by :meth:`fromtimestamp`, and then it's possible to have two timestamps differing by a second that yield - identical :class:`.datetime` objects. See also :meth:`utcfromtimestamp`. + identical :class:`.datetime` objects. This method is preferred over + :meth:`utcfromtimestamp`. .. versionchanged:: 3.3 Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp @@ -789,7 +923,9 @@ Other constructors, all class methods: .. classmethod:: datetime.utcfromtimestamp(timestamp) Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with - :attr:`.tzinfo` ``None``. This may raise :exc:`OverflowError`, if the timestamp is + :attr:`.tzinfo` ``None``. (The resulting object is naive.) + + This may raise :exc:`OverflowError`, if the timestamp is out of the range of values supported by the platform C :c:func:`gmtime` function, and :exc:`OSError` on :c:func:`gmtime` failure. It's common for this to be restricted to years in 1970 through 2038. @@ -806,6 +942,14 @@ Other constructors, all class methods: except the latter formula always supports the full years range: between :const:`MINYEAR` and :const:`MAXYEAR` inclusive. + .. warning:: + + Because naive ``datetime`` objects are treated by many ``datetime`` methods + as local times, it is preferred to use aware datetimes to represent times + in UTC. As such, the recommended way to create an object representing a + specific timestamp in UTC is by calling + ``datetime.fromtimestamp(timestamp, tz=timezone.utc)``. + .. versionchanged:: 3.3 Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp is out of the range of values supported by the platform C @@ -817,7 +961,7 @@ Other constructors, all class methods: Return the :class:`.datetime` corresponding to the proleptic Gregorian ordinal, where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 - <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and + <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and microsecond of the result are all 0, and :attr:`.tzinfo` is ``None``. @@ -825,13 +969,13 @@ Other constructors, all class methods: Return a new :class:`.datetime` object whose date components are equal to the given :class:`date` object's, and whose time components - are equal to the given :class:`.time` object's. If the *tzinfo* + are equal to the given :class:`.time` object's. If the *tzinfo* argument is provided, its value is used to set the :attr:`.tzinfo` attribute of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument is used. For any :class:`.datetime` object *d*, - ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. If date is a + ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. If date is a :class:`.datetime` object, its time components and :attr:`.tzinfo` attributes are ignored. @@ -841,24 +985,60 @@ Other constructors, all class methods: .. classmethod:: datetime.fromisoformat(date_string) - Return a :class:`datetime` corresponding to a *date_string* in one of the - formats emitted by :meth:`date.isoformat` and :meth:`datetime.isoformat`. - Specifically, this function supports strings in the format(s) - ``YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]]``, - where ``*`` can match any single character. + Return a :class:`.datetime` corresponding to a *date_string* in one of the + formats emitted by :meth:`date.isoformat` and :meth:`datetime.isoformat`. + + Specifically, this function supports strings in the format: + + .. code-block:: none + + YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]] + + where ``*`` can match any single character. + + .. caution:: + + This does *not* support parsing arbitrary ISO 8601 strings - it is only intended + as the inverse operation of :meth:`datetime.isoformat`. A more full-featured + ISO 8601 parser, ``dateutil.parser.isoparse`` is available in the third-party package + `dateutil `__. - .. caution:: + Examples:: - This does not support parsing arbitrary ISO 8601 strings - it is only intended - as the inverse operation of :meth:`datetime.isoformat`. + >>> from datetime import datetime + >>> datetime.fromisoformat('2011-11-04') + datetime.datetime(2011, 11, 4, 0, 0) + >>> datetime.fromisoformat('2011-11-04T00:05:23') + datetime.datetime(2011, 11, 4, 0, 5, 23) + >>> datetime.fromisoformat('2011-11-04 00:05:23.283') + datetime.datetime(2011, 11, 4, 0, 5, 23, 283000) + >>> datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') + datetime.datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=datetime.timezone.utc) + >>> datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE + datetime.datetime(2011, 11, 4, 0, 5, 23, + tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) - .. versionadded:: 3.7 + .. versionadded:: 3.7 + +.. classmethod:: datetime.fromisocalendar(year, week, day) + + Return a :class:`.datetime` corresponding to the ISO calendar date specified + by year, week and day. The non-date components of the datetime are populated + with their normal default values. This is the inverse of the function + :meth:`datetime.isocalendar`. + + .. versionadded:: 3.8 .. classmethod:: datetime.strptime(date_string, format) Return a :class:`.datetime` corresponding to *date_string*, parsed according to - *format*. This is equivalent to ``datetime(*(time.strptime(date_string, - format)[0:6]))``. :exc:`ValueError` is raised if the date_string and format + *format*. + + This is equivalent to:: + + datetime(*(time.strptime(date_string, format)[0:6])) + + :exc:`ValueError` is raised if the date_string and format can't be parsed by :func:`time.strptime` or if it returns a value which isn't a time tuple. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. @@ -930,7 +1110,7 @@ Instance attributes (read-only): .. attribute:: datetime.fold - In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A + In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A repeated interval occurs when clocks are rolled back at the end of daylight saving time or when the UTC offset for the current zone is decreased for political reasons.) The value 0 (1) represents the earlier (later) of the two moments with the same wall @@ -955,7 +1135,7 @@ Supported operations: (1) datetime2 is a duration of timedelta removed from datetime1, moving forward in - time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The + time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and datetime2 - datetime1 == timedelta after. :exc:`OverflowError` is raised if datetime2.year would be smaller than :const:`MINYEAR` or larger than @@ -969,16 +1149,16 @@ Supported operations: (3) Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if - both operands are naive, or if both are aware. If one is aware and the other is + both operands are naive, or if both are aware. If one is aware and the other is naive, :exc:`TypeError` is raised. If both are naive, or both are aware and have the same :attr:`~.datetime.tzinfo` attribute, the :attr:`~.datetime.tzinfo` attributes are ignored, and the result is a :class:`timedelta` - object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments + object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments are done in this case. If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes first. The + as if *a* and *b* were first converted to naive UTC datetimes first. The result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - b.utcoffset())`` except that the implementation never overflows. @@ -987,34 +1167,31 @@ Supported operations: *datetime2* in time. If one comparand is naive and the other is aware, :exc:`TypeError` - is raised if an order comparison is attempted. For equality + is raised if an order comparison is attempted. For equality comparisons, naive instances are never equal to aware instances. If both comparands are aware, and have the same :attr:`~.datetime.tzinfo` attribute, the common :attr:`~.datetime.tzinfo` attribute is ignored and the base datetimes are - compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` + compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparands are first adjusted by subtracting their UTC offsets (obtained from ``self.utcoffset()``). .. versionchanged:: 3.3 - Equality comparisons between naive and aware :class:`.datetime` + Equality comparisons between aware and naive :class:`.datetime` instances don't raise :exc:`TypeError`. .. note:: In order to stop comparison from falling back to the default scheme of comparing object addresses, datetime comparison normally raises :exc:`TypeError` if the - other comparand isn't also a :class:`.datetime` object. However, + other comparand isn't also a :class:`.datetime` object. However, ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`.datetime` + :meth:`timetuple` attribute. This hook gives other kinds of date objects a + chance at implementing mixed-type comparison. If not, when a :class:`.datetime` object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return + unless the comparison is ``==`` or ``!=``. The latter cases return :const:`False` or :const:`True`, respectively. -:class:`.datetime` objects can be used as dictionary keys. In Boolean contexts, -all :class:`.datetime` objects are considered to be true. - Instance methods: .. method:: datetime.date() @@ -1025,7 +1202,7 @@ Instance methods: .. method:: datetime.time() Return :class:`.time` object with same hour, minute, second, microsecond and fold. - :attr:`.tzinfo` is ``None``. See also method :meth:`timetz`. + :attr:`.tzinfo` is ``None``. See also method :meth:`timetz`. .. versionchanged:: 3.6 The fold value is copied to the returned :class:`.time` object. @@ -1034,7 +1211,7 @@ Instance methods: .. method:: datetime.timetz() Return :class:`.time` object with same hour, minute, second, microsecond, fold, and - tzinfo attributes. See also method :meth:`time`. + tzinfo attributes. See also method :meth:`time`. .. versionchanged:: 3.6 The fold value is copied to the returned :class:`.time` object. @@ -1045,7 +1222,7 @@ Instance methods: tzinfo=self.tzinfo, * fold=0) Return a datetime with the same attributes, except for those attributes given - new values by whichever keyword arguments are specified. Note that + new values by whichever keyword arguments are specified. Note that ``tzinfo=None`` can be specified to create a naive datetime from an aware datetime with no conversion of date and time data. @@ -1060,11 +1237,11 @@ Instance methods: *self*, but in *tz*'s local time. If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its - :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self* + :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self* is naive, it is presumed to represent time in the system timezone. If called without arguments (or with ``tz=None``) the system local - timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted + timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted datetime instance will be set to an instance of :class:`timezone` with the zone name and offset obtained from the OS. @@ -1075,7 +1252,7 @@ Instance methods: the same date and time data as ``dt - dt.utcoffset()``. If you merely want to attach a time zone object *tz* to a datetime *dt* without - adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you + adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you merely want to remove the time zone object from an aware datetime *dt* without conversion of date and time data, use ``dt.replace(tzinfo=None)``. @@ -1129,44 +1306,58 @@ Instance methods: .. method:: datetime.timetuple() Return a :class:`time.struct_time` such as returned by :func:`time.localtime`. - ``d.timetuple()`` is equivalent to ``time.struct_time((d.year, d.month, d.day, - d.hour, d.minute, d.second, d.weekday(), yday, dst))``, where ``yday = - d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` is the day number within - the current year starting with ``1`` for January 1st. The :attr:`tm_isdst` flag - of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is - ``None`` or :meth:`dst` returns ``None``, :attr:`tm_isdst` is set to ``-1``; - else if :meth:`dst` returns a non-zero value, :attr:`tm_isdst` is set to ``1``; - else :attr:`tm_isdst` is set to ``0``. + + ``d.timetuple()`` is equivalent to:: + + time.struct_time((d.year, d.month, d.day, + d.hour, d.minute, d.second, + d.weekday(), yday, dst)) + + where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` + is the day number within the current year starting with ``1`` for January + 1st. The :attr:`tm_isdst` flag of the result is set according to the + :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns + ``None``, :attr:`tm_isdst` is set to ``-1``; else if :meth:`dst` returns a + non-zero value, :attr:`tm_isdst` is set to ``1``; else :attr:`tm_isdst` is + set to ``0``. .. method:: datetime.utctimetuple() If :class:`.datetime` instance *d* is naive, this is the same as ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what - ``d.dst()`` returns. DST is never in effect for a UTC time. + ``d.dst()`` returns. DST is never in effect for a UTC time. If *d* is aware, *d* is normalized to UTC time, by subtracting ``d.utcoffset()``, and a :class:`time.struct_time` for the - normalized time is returned. :attr:`tm_isdst` is forced to 0. Note + normalized time is returned. :attr:`tm_isdst` is forced to 0. Note that an :exc:`OverflowError` may be raised if *d*.year was ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year boundary. + .. warning:: + + Because naive ``datetime`` objects are treated by many ``datetime`` methods + as local times, it is preferred to use aware datetimes to represent times + in UTC; as a result, using ``utcfromtimetuple`` may give misleading + results. If you have a naive ``datetime`` representing UTC, use + ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point + you can use :meth:`.datetime.timetuple`. .. method:: datetime.toordinal() - Return the proleptic Gregorian ordinal of the date. The same as + Return the proleptic Gregorian ordinal of the date. The same as ``self.date().toordinal()``. .. method:: datetime.timestamp() Return POSIX timestamp corresponding to the :class:`.datetime` - instance. The return value is a :class:`float` similar to that + instance. The return value is a :class:`float` similar to that returned by :func:`time.time`. Naive :class:`.datetime` instances are assumed to represent local time and this method relies on the platform C :c:func:`mktime` - function to perform the conversion. Since :class:`.datetime` + function to perform the conversion. Since :class:`.datetime` supports wider range of values than :c:func:`mktime` on many platforms, this method may raise :exc:`OverflowError` for times far in the past or far in the future. @@ -1185,7 +1376,7 @@ Instance methods: .. note:: There is no method to obtain the POSIX timestamp directly from a - naive :class:`.datetime` instance representing UTC time. If your + naive :class:`.datetime` instance representing UTC time. If your application uses this convention and your system timezone is not set to UTC, you can obtain the POSIX timestamp by supplying ``tzinfo=timezone.utc``:: @@ -1211,30 +1402,45 @@ Instance methods: .. method:: datetime.isocalendar() - Return a 3-tuple, (ISO year, ISO week number, ISO weekday). The same as - ``self.date().isocalendar()``. + Return a :term:`named tuple` with three components: ``year``, ``week`` + and ``weekday``. The same as ``self.date().isocalendar()``. .. method:: datetime.isoformat(sep='T', timespec='auto') - Return a string representing the date and time in ISO 8601 format, - YYYY-MM-DDTHH:MM:SS.ffffff or, if :attr:`microsecond` is 0, - YYYY-MM-DDTHH:MM:SS + Return a string representing the date and time in ISO 8601 format: + + - ``YYYY-MM-DDTHH:MM:SS.ffffff``, if :attr:`microsecond` is not 0 + - ``YYYY-MM-DDTHH:MM:SS``, if :attr:`microsecond` is 0 If :meth:`utcoffset` does not return ``None``, a string is appended, giving the UTC offset: - YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] or, if :attr:`microsecond` - is 0 YYYY-MM-DDTHH:MM:SS+HH:MM[:SS[.ffffff]]. + + - ``YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` + is not 0 + - ``YYYY-MM-DDTHH:MM:SS+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` is 0 + + Examples:: + + >>> from datetime import datetime, timezone + >>> datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() + '2019-05-18T15:17:08.132263' + >>> datetime(2019, 5, 18, 15, 17, tzinfo=timezone.utc).isoformat() + '2019-05-18T15:17:00+00:00' The optional argument *sep* (default ``'T'``) is a one-character separator, - placed between the date and time portions of the result. For example, + placed between the date and time portions of the result. For example:: >>> from datetime import tzinfo, timedelta, datetime >>> class TZ(tzinfo): - ... def utcoffset(self, dt): return timedelta(minutes=-399) + ... """A time zone with an arbitrary, constant -06:39 offset.""" + ... def utcoffset(self, dt): + ... return timedelta(hours=-6, minutes=-39) ... >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') '2002-12-25 00:00:00-06:39' + >>> datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() + '2009-11-27T00:00:00.000100-06:39' The optional argument *timespec* specifies the number of additional components of the time to include (the default is ``'auto'``). @@ -1242,19 +1448,19 @@ Instance methods: - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, same as ``'microseconds'`` otherwise. - - ``'hours'``: Include the :attr:`hour` in the two-digit HH format. - - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format. + - ``'hours'``: Include the :attr:`hour` in the two-digit ``HH`` format. + - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in ``HH:MM`` format. - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` - in HH:MM:SS format. + in ``HH:MM:SS`` format. - ``'milliseconds'``: Include full time, but truncate fractional second - part to milliseconds. HH:MM:SS.sss format. - - ``'microseconds'``: Include full time in HH:MM:SS.ffffff format. + part to milliseconds. ``HH:MM:SS.sss`` format. + - ``'microseconds'``: Include full time in ``HH:MM:SS.ffffff`` format. .. note:: Excluded time components are truncated, not rounded. - :exc:`ValueError` will be raised on an invalid *timespec* argument. + :exc:`ValueError` will be raised on an invalid *timespec* argument:: >>> from datetime import datetime @@ -1276,48 +1482,64 @@ Instance methods: .. method:: datetime.ctime() - Return a string representing the date and time, for example ``datetime(2002, 12, - 4, 20, 30, 40).ctime() == 'Wed Dec 4 20:30:40 2002'``. ``d.ctime()`` is - equivalent to ``time.ctime(time.mktime(d.timetuple()))`` on platforms where the - native C :c:func:`ctime` function (which :func:`time.ctime` invokes, but which - :meth:`datetime.ctime` does not invoke) conforms to the C standard. + Return a string representing the date and time:: + + >>> from datetime import datetime + >>> datetime(2002, 12, 4, 20, 30, 40).ctime() + 'Wed Dec 4 20:30:40 2002' + + The output string will *not* include time zone information, regardless + of whether the input is aware or naive. + ``d.ctime()`` is equivalent to:: + + time.ctime(time.mktime(d.timetuple())) + + on platforms where the native C :c:func:`ctime` function + (which :func:`time.ctime` invokes, but which + :meth:`datetime.ctime` does not invoke) conforms to the C standard. .. method:: datetime.strftime(format) Return a string representing the date and time, controlled by an explicit format - string. For a complete list of formatting directives, see + string. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. .. method:: datetime.__format__(format) - Same as :meth:`.datetime.strftime`. This makes it possible to specify a format + Same as :meth:`.datetime.strftime`. This makes it possible to specify a format string for a :class:`.datetime` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. For a + literals ` and when using :meth:`str.format`. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. +Examples of Usage: :class:`.datetime` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Examples of working with datetime objects: +Examples of working with :class:`~datetime.datetime` objects: .. doctest:: - >>> from datetime import datetime, date, time + >>> from datetime import datetime, date, time, timezone + >>> # Using datetime.combine() >>> d = date(2005, 7, 14) >>> t = time(12, 30) >>> datetime.combine(d, t) datetime.datetime(2005, 7, 14, 12, 30) - >>> # Using datetime.now() or datetime.utcnow() + + >>> # Using datetime.now() >>> datetime.now() # doctest: +SKIP datetime.datetime(2007, 12, 6, 16, 29, 43, 79043) # GMT +1 - >>> datetime.utcnow() # doctest: +SKIP - datetime.datetime(2007, 12, 6, 15, 29, 43, 79060) + >>> datetime.now(timezone.utc) # doctest: +SKIP + datetime.datetime(2007, 12, 6, 15, 29, 43, 79060, tzinfo=datetime.timezone.utc) + >>> # Using datetime.strptime() >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") >>> dt datetime.datetime(2006, 11, 21, 16, 30) + >>> # Using datetime.timetuple() to get tuple of all attributes >>> tt = dt.timetuple() >>> for it in tt: # doctest: +SKIP @@ -1332,6 +1554,7 @@ Examples of working with datetime objects: 1 # weekday (0 = Monday) 325 # number of days since 1st January -1 # dst - method tzinfo.dst() returned None + >>> # Date in ISO format >>> ic = dt.isocalendar() >>> for it in ic: # doctest: +SKIP @@ -1340,81 +1563,95 @@ Examples of working with datetime objects: 2006 # ISO year 47 # ISO week 2 # ISO weekday - >>> # Formatting datetime + + >>> # Formatting a datetime >>> dt.strftime("%A, %d. %B %Y %I:%M%p") 'Tuesday, 21. November 2006 04:30PM' >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") 'The day is 21, the month is November, the time is 04:30PM.' -Using datetime with tzinfo: - - >>> from datetime import timedelta, datetime, tzinfo - >>> class GMT1(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=1) + self.dst(dt) - ... def dst(self, dt): - ... # DST starts last Sunday in March - ... d = datetime(dt.year, 4, 1) # ends last Sunday in October - ... self.dston = d - timedelta(days=d.weekday() + 1) - ... d = datetime(dt.year, 11, 1) - ... self.dstoff = d - timedelta(days=d.weekday() + 1) - ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) - ... else: - ... return timedelta(0) - ... def tzname(self,dt): - ... return "GMT +1" - ... - >>> class GMT2(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=2) + self.dst(dt) - ... def dst(self, dt): - ... d = datetime(dt.year, 4, 1) - ... self.dston = d - timedelta(days=d.weekday() + 1) - ... d = datetime(dt.year, 11, 1) - ... self.dstoff = d - timedelta(days=d.weekday() + 1) - ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) - ... else: - ... return timedelta(0) - ... def tzname(self,dt): - ... return "GMT +2" - ... - >>> gmt1 = GMT1() - >>> # Daylight Saving Time - >>> dt1 = datetime(2006, 11, 21, 16, 30, tzinfo=gmt1) - >>> dt1.dst() - datetime.timedelta(0) - >>> dt1.utcoffset() - datetime.timedelta(seconds=3600) - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=gmt1) - >>> dt2.dst() - datetime.timedelta(seconds=3600) - >>> dt2.utcoffset() - datetime.timedelta(seconds=7200) - >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(GMT2()) - >>> dt3 # doctest: +ELLIPSIS - datetime.datetime(2006, 6, 14, 14, 0, tzinfo=) - >>> dt2 # doctest: +ELLIPSIS - datetime.datetime(2006, 6, 14, 13, 0, tzinfo=) - >>> dt2.utctimetuple() == dt3.utctimetuple() - True - - +The example below defines a :class:`tzinfo` subclass capturing time zone +information for Kabul, Afghanistan, which used +4 UTC until 1945 +and then +4:30 UTC thereafter:: + + from datetime import timedelta, datetime, tzinfo, timezone + + class KabulTz(tzinfo): + # Kabul used +4 until 1945, when they moved to +4:30 + UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) + + def utcoffset(self, dt): + if dt.year < 1945: + return timedelta(hours=4) + elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): + # An ambiguous ("imaginary") half-hour range representing + # a 'fold' in time due to the shift from +4 to +4:30. + # If dt falls in the imaginary range, use fold to decide how + # to resolve. See PEP495. + return timedelta(hours=4, minutes=(30 if dt.fold else 0)) + else: + return timedelta(hours=4, minutes=30) + + def fromutc(self, dt): + # Follow same validations as in datetime.tzinfo + if not isinstance(dt, datetime): + raise TypeError("fromutc() requires a datetime argument") + if dt.tzinfo is not self: + raise ValueError("dt.tzinfo is not self") + + # A custom implementation is required for fromutc as + # the input to this function is a datetime with utc values + # but with a tzinfo set to self. + # See datetime.astimezone or fromtimestamp. + if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: + return dt + timedelta(hours=4, minutes=30) + else: + return dt + timedelta(hours=4) + + def dst(self, dt): + # Kabul does not observe daylight saving time. + return timedelta(0) + + def tzname(self, dt): + if dt >= self.UTC_MOVE_DATE: + return "+04:30" + return "+04" + +Usage of ``KabulTz`` from above:: + + >>> tz1 = KabulTz() + + >>> # Datetime before the change + >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) + >>> print(dt1.utcoffset()) + 4:00:00 + + >>> # Datetime after the change + >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) + >>> print(dt2.utcoffset()) + 4:30:00 + + >>> # Convert datetime to another time zone + >>> dt3 = dt2.astimezone(timezone.utc) + >>> dt3 + datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) + >>> dt2 + datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz()) + >>> dt2 == dt3 + True .. _datetime-time: :class:`.time` Objects ---------------------- -A time object represents a (local) time of day, independent of any particular +A :class:`time` object represents a (local) time of day, independent of any particular day, and subject to adjustment via a :class:`tzinfo` object. .. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) - All arguments are optional. *tzinfo* may be ``None``, or an instance of a - :class:`tzinfo` subclass. The remaining arguments may be integers, in the + All arguments are optional. *tzinfo* may be ``None``, or an instance of a + :class:`tzinfo` subclass. The remaining arguments must be integers in the following ranges: * ``0 <= hour < 24``, @@ -1423,7 +1660,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. * ``0 <= microsecond < 1000000``, * ``fold in [0, 1]``. - If an argument outside those ranges is given, :exc:`ValueError` is raised. All + If an argument outside those ranges is given, :exc:`ValueError` is raised. All default to ``0`` except *tzinfo*, which defaults to :const:`None`. Class attributes: @@ -1476,7 +1713,7 @@ Instance attributes (read-only): .. attribute:: time.fold - In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A + In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A repeated interval occurs when clocks are rolled back at the end of daylight saving time or when the UTC offset for the current zone is decreased for political reasons.) The value 0 (1) represents the earlier (later) of the two moments with the same wall @@ -1484,38 +1721,32 @@ Instance attributes (read-only): .. versionadded:: 3.6 +:class:`.time` objects support comparison of :class:`.time` to :class:`.time`, +where *a* is considered less +than *b* when *a* precedes *b* in time. If one comparand is naive and the other +is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality +comparisons, naive instances are never equal to aware instances. -Supported operations: - -* comparison of :class:`.time` to :class:`.time`, where *a* is considered less - than *b* when *a* precedes *b* in time. If one comparand is naive and the other - is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. - - If both comparands are aware, and have - the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is - ignored and the base times are compared. If both comparands are aware and - have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by - subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order - to stop mixed-type comparisons from falling back to the default comparison by - object address, when a :class:`.time` object is compared to an object of a - different type, :exc:`TypeError` is raised unless the comparison is ``==`` or - ``!=``. The latter cases return :const:`False` or :const:`True`, respectively. - - .. versionchanged:: 3.3 - Equality comparisons between naive and aware :class:`~datetime.time` instances - don't raise :exc:`TypeError`. +If both comparands are aware, and have +the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is +ignored and the base times are compared. If both comparands are aware and +have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by +subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order +to stop mixed-type comparisons from falling back to the default comparison by +object address, when a :class:`.time` object is compared to an object of a +different type, :exc:`TypeError` is raised unless the comparison is ``==`` or +``!=``. The latter cases return :const:`False` or :const:`True`, respectively. -* hash, use as dict key +.. versionchanged:: 3.3 + Equality comparisons between aware and naive :class:`~datetime.time` instances + don't raise :exc:`TypeError`. -* efficient pickling - -In boolean contexts, a :class:`.time` object is always considered to be true. +In Boolean contexts, a :class:`.time` object is always considered to be true. .. versionchanged:: 3.5 Before Python 3.5, a :class:`.time` object was considered to be false if it - represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full + represented midnight in UTC. This behavior was considered obscure and + error-prone and has been removed in Python 3.5. See :issue:`13936` for full details. @@ -1523,16 +1754,30 @@ Other constructor: .. classmethod:: time.fromisoformat(time_string) - Return a :class:`time` corresponding to a *time_string* in one of the - formats emitted by :meth:`time.isoformat`. Specifically, this function supports - strings in the format(s) ``HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]``. + Return a :class:`.time` corresponding to a *time_string* in one of the + formats emitted by :meth:`time.isoformat`. Specifically, this function supports + strings in the format: + + .. code-block:: none + + HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]] + + .. caution:: + + This does *not* support parsing arbitrary ISO 8601 strings. It is only + intended as the inverse operation of :meth:`time.isoformat`. - .. caution:: + Examples:: - This does not support parsing arbitrary ISO 8601 strings - it is only intended - as the inverse operation of :meth:`time.isoformat`. + >>> from datetime import time + >>> time.fromisoformat('04:23:01') + datetime.time(4, 23, 1) + >>> time.fromisoformat('04:23:01.000384') + datetime.time(4, 23, 1, 384) + >>> time.fromisoformat('04:23:01+04:00') + datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) - .. versionadded:: 3.7 + .. versionadded:: 3.7 Instance methods: @@ -1541,7 +1786,7 @@ Instance methods: microsecond=self.microsecond, tzinfo=self.tzinfo, * fold=0) Return a :class:`.time` with the same value, except for those attributes given - new values by whichever keyword arguments are specified. Note that + new values by whichever keyword arguments are specified. Note that ``tzinfo=None`` can be specified to create a naive :class:`.time` from an aware :class:`.time`, without conversion of the time data. @@ -1551,10 +1796,12 @@ Instance methods: .. method:: time.isoformat(timespec='auto') - Return a string representing the time in ISO 8601 format, HH:MM:SS.ffffff or, if - :attr:`microsecond` is 0, HH:MM:SS If :meth:`utcoffset` does not return ``None``, a - string is appended, giving the UTC offset: HH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] - or, if self.microsecond is 0, HH:MM:SS+HH:MM[:SS[.ffffff]]. + Return a string representing the time in ISO 8601 format, one of: + + - ``HH:MM:SS.ffffff``, if :attr:`microsecond` is not 0 + - ``HH:MM:SS``, if :attr:`microsecond` is 0 + - ``HH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]``, if :meth:`utcoffset` does not return ``None`` + - ``HH:MM:SS+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` is 0 and :meth:`utcoffset` does not return ``None`` The optional argument *timespec* specifies the number of additional components of the time to include (the default is ``'auto'``). @@ -1562,13 +1809,13 @@ Instance methods: - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, same as ``'microseconds'`` otherwise. - - ``'hours'``: Include the :attr:`hour` in the two-digit HH format. - - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format. + - ``'hours'``: Include the :attr:`hour` in the two-digit ``HH`` format. + - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in ``HH:MM`` format. - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` - in HH:MM:SS format. + in ``HH:MM:SS`` format. - ``'milliseconds'``: Include full time, but truncate fractional second - part to milliseconds. HH:MM:SS.sss format. - - ``'microseconds'``: Include full time in HH:MM:SS.ffffff format. + part to milliseconds. ``HH:MM:SS.sss`` format. + - ``'microseconds'``: Include full time in ``HH:MM:SS.ffffff`` format. .. note:: @@ -1576,6 +1823,7 @@ Instance methods: :exc:`ValueError` will be raised on an invalid *timespec* argument. + Example:: >>> from datetime import time >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') @@ -1598,7 +1846,7 @@ Instance methods: .. method:: time.strftime(format) Return a string representing the time, controlled by an explicit format - string. For a complete list of formatting directives, see + string. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. @@ -1606,7 +1854,7 @@ Instance methods: Same as :meth:`.time.strftime`. This makes it possible to specify a format string for a :class:`.time` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. For a + literals ` and when using :meth:`str.format`. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. @@ -1636,29 +1884,33 @@ Instance methods: ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't return ``None`` or a string object. -Example: +Examples of Usage: :class:`.time` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of working with a :class:`.time` object:: >>> from datetime import time, tzinfo, timedelta - >>> class GMT1(tzinfo): + >>> class TZ1(tzinfo): ... def utcoffset(self, dt): ... return timedelta(hours=1) ... def dst(self, dt): ... return timedelta(0) ... def tzname(self,dt): - ... return "Europe/Prague" + ... return "+01:00" + ... def __repr__(self): + ... return f"{self.__class__.__name__}()" ... - >>> t = time(12, 10, 30, tzinfo=GMT1()) - >>> t # doctest: +ELLIPSIS - datetime.time(12, 10, 30, tzinfo=) - >>> gmt = GMT1() + >>> t = time(12, 10, 30, tzinfo=TZ1()) + >>> t + datetime.time(12, 10, 30, tzinfo=TZ1()) >>> t.isoformat() '12:10:30+01:00' >>> t.dst() datetime.timedelta(0) >>> t.tzname() - 'Europe/Prague' + '+01:00' >>> t.strftime("%H:%M:%S %Z") - '12:10:30 Europe/Prague' + '12:10:30 +01:00' >>> 'The {} is {:%H:%M}.'.format("time", t) 'The time is 12:10.' @@ -1671,12 +1923,8 @@ Example: .. class:: tzinfo() This is an abstract base class, meaning that this class should not be - instantiated directly. You need to derive a concrete subclass, and (at least) - supply implementations of the standard :class:`tzinfo` methods needed by the - :class:`.datetime` methods you use. The :mod:`datetime` module supplies - a simple concrete subclass of :class:`tzinfo`, :class:`timezone`, which can represent - timezones with fixed offset from UTC such as UTC itself or North American EST and - EDT. + instantiated directly. Define a subclass of :class:`tzinfo` to capture + information about a particular time zone. An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the constructors for :class:`.datetime` and :class:`.time` objects. The latter objects @@ -1684,28 +1932,35 @@ Example: supports methods revealing offset of local time from UTC, the name of the time zone, and DST offset, all relative to a date or time object passed to them. + You need to derive a concrete subclass, and (at least) + supply implementations of the standard :class:`tzinfo` methods needed by the + :class:`.datetime` methods you use. The :mod:`datetime` module provides + :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can + represent timezones with fixed offset from UTC such as UTC itself or North + American EST and EDT. + Special requirement for pickling: A :class:`tzinfo` subclass must have an - :meth:`__init__` method that can be called with no arguments, else it can be - pickled but possibly not unpickled again. This is a technical requirement that + :meth:`__init__` method that can be called with no arguments, otherwise it can be + pickled but possibly not unpickled again. This is a technical requirement that may be relaxed in the future. A concrete subclass of :class:`tzinfo` may need to implement the following - methods. Exactly which methods are needed depends on the uses made of aware - :mod:`datetime` objects. If in doubt, simply implement all of them. + methods. Exactly which methods are needed depends on the uses made of aware + :mod:`datetime` objects. If in doubt, simply implement all of them. .. method:: tzinfo.utcoffset(dt) Return offset of local time from UTC, as a :class:`timedelta` object that is - positive east of UTC. If local time is - west of UTC, this should be negative. Note that this is intended to be the - total offset from UTC; for example, if a :class:`tzinfo` object represents both - time zone and DST adjustments, :meth:`utcoffset` should return their sum. If - the UTC offset isn't known, return ``None``. Else the value returned must be a - :class:`timedelta` object strictly between ``-timedelta(hours=24)`` and - ``timedelta(hours=24)`` (the magnitude of the offset must be less - than one day). Most implementations of :meth:`utcoffset` will probably look - like one of these two:: + positive east of UTC. If local time is west of UTC, this should be negative. + + This represents the *total* offset from UTC; for example, if a + :class:`tzinfo` object represents both time zone and DST adjustments, + :meth:`utcoffset` should return their sum. If the UTC offset isn't known, + return ``None``. Else the value returned must be a :class:`timedelta` object + strictly between ``-timedelta(hours=24)`` and ``timedelta(hours=24)`` + (the magnitude of the offset must be less than one day). Most implementations + of :meth:`utcoffset` will probably look like one of these two:: return CONSTANT # fixed-offset class return CONSTANT + self.dst(dt) # daylight-aware class @@ -1724,12 +1979,14 @@ Example: Return the daylight saving time (DST) adjustment, as a :class:`timedelta` object or - ``None`` if DST information isn't known. Return ``timedelta(0)`` if DST is not - in effect. If DST is in effect, return the offset as a :class:`timedelta` object + ``None`` if DST information isn't known. + + Return ``timedelta(0)`` if DST is not in effect. + If DST is in effect, return the offset as a :class:`timedelta` object (see :meth:`utcoffset` for details). Note that DST offset, if applicable, has already been added to the UTC offset returned by :meth:`utcoffset`, so there's no need to consult :meth:`dst` unless you're interested in obtaining DST info - separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo` + separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo` attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for DST changes when crossing time zones. @@ -1742,9 +1999,9 @@ Example: must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == tz`` For sane :class:`tzinfo` subclasses, this expression yields the time zone's "standard offset", which should not depend on the date or the time, but - only on geographic location. The implementation of :meth:`datetime.astimezone` + only on geographic location. The implementation of :meth:`datetime.astimezone` relies on this, but cannot detect violations; it's the programmer's - responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee + responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless. @@ -1754,12 +2011,12 @@ Example: # a fixed-offset class: doesn't account for DST return timedelta(0) - or :: + or:: def dst(self, dt): # Code to set dston and dstoff to the time zone's DST # transition times based on the input dt.year, and expressed - # in standard local time. Then + # in standard local time. if dston <= dt.replace(tzinfo=None) < dstoff: return timedelta(hours=1) @@ -1776,9 +2033,9 @@ Example: Return the time zone name corresponding to the :class:`.datetime` object *dt*, as a string. Nothing about string names is defined by the :mod:`datetime` module, - and there's no requirement that it mean anything in particular. For example, + and there's no requirement that it mean anything in particular. For example, "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all - valid replies. Return ``None`` if a string name isn't known. Note that this is + valid replies. Return ``None`` if a string name isn't known. Note that this is a method rather than a fixed string primarily because some :class:`tzinfo` subclasses will wish to return different names depending on the specific value of *dt* passed, especially if the :class:`tzinfo` class is accounting for @@ -1788,20 +2045,20 @@ Example: These methods are called by a :class:`.datetime` or :class:`.time` object, in -response to their methods of the same names. A :class:`.datetime` object passes +response to their methods of the same names. A :class:`.datetime` object passes itself as the argument, and a :class:`.time` object passes ``None`` as the -argument. A :class:`tzinfo` subclass's methods should therefore be prepared to +argument. A :class:`tzinfo` subclass's methods should therefore be prepared to accept a *dt* argument of ``None``, or of class :class:`.datetime`. When ``None`` is passed, it's up to the class designer to decide the best -response. For example, returning ``None`` is appropriate if the class wishes to -say that time objects don't participate in the :class:`tzinfo` protocols. It +response. For example, returning ``None`` is appropriate if the class wishes to +say that time objects don't participate in the :class:`tzinfo` protocols. It may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as there is no other convention for discovering the standard offset. When a :class:`.datetime` object is passed in response to a :class:`.datetime` -method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can -rely on this, unless user code calls :class:`tzinfo` methods directly. The +method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can +rely on this, unless user code calls :class:`tzinfo` methods directly. The intent is that the :class:`tzinfo` methods interpret *dt* as being in local time, and not need worry about objects in other timezones. @@ -1811,16 +2068,16 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: .. method:: tzinfo.fromutc(dt) This is called from the default :class:`datetime.astimezone()` - implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s - date and time data are to be viewed as expressing a UTC time. The purpose + implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s + date and time data are to be viewed as expressing a UTC time. The purpose of :meth:`fromutc` is to adjust the date and time data, returning an equivalent datetime in *self*'s local time. Most :class:`tzinfo` subclasses should be able to inherit the default - :meth:`fromutc` implementation without problems. It's strong enough to handle + :meth:`fromutc` implementation without problems. It's strong enough to handle fixed-offset time zones, and time zones accounting for both standard and daylight time, and the latter even if the DST transition times differ in - different years. An example of a time zone the default :meth:`fromutc` + different years. An example of a time zone the default :meth:`fromutc` implementation may not handle correctly in all cases is one where the standard offset (from UTC) depends on the specific date and time passed, which can happen for political reasons. The default implementations of :meth:`astimezone` and @@ -1853,7 +2110,7 @@ In the following :download:`tzinfo_examples.py Note that there are unavoidable subtleties twice per year in a :class:`tzinfo` subclass accounting for both standard and daylight time, at the DST transition -points. For concreteness, consider US Eastern (UTC -0500), where EDT begins the +points. For concreteness, consider US Eastern (UTC -0500), where EDT begins the minute after 1:59 (EST) on the second Sunday in March, and ends the minute after 1:59 (EDT) on the first Sunday in November:: @@ -1866,9 +2123,9 @@ minute after 1:59 (EST) on the second Sunday in March, and ends the minute after end 23:MM 0:MM 1:MM 1:MM 2:MM 3:MM When DST starts (the "start" line), the local wall clock leaps from 1:59 to -3:00. A wall time of the form 2:MM doesn't really make sense on that day, so +3:00. A wall time of the form 2:MM doesn't really make sense on that day, so ``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST -begins. For example, at the Spring forward transition of 2016, we get +begins. For example, at the Spring forward transition of 2016, we get:: >>> from datetime import datetime, timezone >>> from tzinfo_examples import HOUR, Eastern @@ -1886,14 +2143,14 @@ begins. For example, at the Spring forward transition of 2016, we get When DST ends (the "end" line), there's a potentially worse problem: there's an hour that can't be spelled unambiguously in local wall time: the last hour of -daylight time. In Eastern, that's times of the form 5:MM UTC on the day -daylight time ends. The local wall clock leaps from 1:59 (daylight time) back +daylight time. In Eastern, that's times of the form 5:MM UTC on the day +daylight time ends. The local wall clock leaps from 1:59 (daylight time) back to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous. :meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC -hours into the same local hour then. In the Eastern example, UTC times of the +hours into the same local hour then. In the Eastern example, UTC times of the form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1. -For example, at the Fall back transition of 2016, we get +For example, at the Fall back transition of 2016, we get:: >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) >>> for i in range(4): @@ -1906,7 +2163,7 @@ For example, at the Fall back transition of 2016, we get 06:00:00 UTC = 01:00:00 EST 1 07:00:00 UTC = 02:00:00 EST 0 -Note that the :class:`datetime` instances that differ only by the value of the +Note that the :class:`.datetime` instances that differ only by the value of the :attr:`~datetime.fold` attribute are considered equal in comparisons. Applications that can't bear wall-time ambiguities should explicitly check the @@ -1918,15 +2175,17 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: `dateutil.tz `_ - The standard library has :class:`timezone` class for handling arbitrary - fixed offsets from UTC and :attr:`timezone.utc` as UTC timezone instance. + The :mod:`datetime` module has a basic :class:`timezone` class (for + handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` + attribute (a UTC timezone instance). - *dateutil.tz* library brings the *IANA timezone database* (also known as the - Olson database) to Python and its usage is recommended. + *dateutil.tz* library brings the *IANA timezone database* + (also known as the Olson database) to Python, and its usage is + recommended. `IANA timezone database `_ - The Time Zone Database (often called tz, tzdata or zoneinfo) contains code and - data that represent the history of local time for many representative + The Time Zone Database (often called tz, tzdata or zoneinfo) contains code + and data that represent the history of local time for many representative locations around the globe. It is updated periodically to reflect changes made by political bodies to time zone boundaries, UTC offsets, and daylight-saving rules. @@ -1939,20 +2198,21 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a timezone defined by a fixed offset from -UTC. Note that objects of this class cannot be used to represent -timezone information in the locations where different offsets are used -in different days of the year or where historical changes have been -made to civil time. +UTC. + +Objects of this class cannot be used to represent timezone information in the +locations where different offsets are used in different days of the year or +where historical changes have been made to civil time. .. class:: timezone(offset, name=None) The *offset* argument must be specified as a :class:`timedelta` - object representing the difference between the local time and UTC. It must + object representing the difference between the local time and UTC. It must be strictly between ``-timedelta(hours=24)`` and ``timedelta(hours=24)``, otherwise :exc:`ValueError` is raised. - The *name* argument is optional. If specified it must be a string that + The *name* argument is optional. If specified it must be a string that will be used as the value returned by the :meth:`datetime.tzname` method. .. versionadded:: 3.2 @@ -1964,9 +2224,10 @@ made to civil time. .. method:: timezone.utcoffset(dt) Return the fixed value specified when the :class:`timezone` instance is - constructed. The *dt* argument is ignored. The return value is a - :class:`timedelta` instance equal to the difference between the - local time and UTC. + constructed. + + The *dt* argument is ignored. The return value is a :class:`timedelta` + instance equal to the difference between the local time and UTC. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. @@ -1974,16 +2235,17 @@ made to civil time. .. method:: timezone.tzname(dt) Return the fixed value specified when the :class:`timezone` instance - is constructed. If *name* is not provided in the constructor, the - name returned by ``tzname(dt)`` is generated from the value of the - ``offset`` as follows. If *offset* is ``timedelta(0)``, the name - is "UTC", otherwise it is a string 'UTC±HH:MM', where ± is the sign - of ``offset``, HH and MM are two digits of ``offset.hours`` and - ``offset.minutes`` respectively. + is constructed. + + If *name* is not provided in the constructor, the name returned by + ``tzname(dt)`` is generated from the value of the ``offset`` as follows. If + *offset* is ``timedelta(0)``, the name is "UTC", otherwise it is a string in + the format ``UTC±HH:MM``, where ± is the sign of ``offset``, HH and MM are + two digits of ``offset.hours`` and ``offset.minutes`` respectively. .. versionchanged:: 3.6 - Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not - 'UTC+00:00'. + Name generated from ``offset=timedelta(0)`` is now plain `'UTC'`, not + ``'UTC+00:00'``. .. method:: timezone.dst(dt) @@ -1992,7 +2254,7 @@ made to civil time. .. method:: timezone.fromutc(dt) - Return ``dt + offset``. The *dt* argument must be an aware + Return ``dt + offset``. The *dt* argument must be an aware :class:`.datetime` instance, with ``tzinfo`` set to ``self``. Class attributes: @@ -2012,40 +2274,33 @@ Class attributes: :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the -control of an explicit format string. Broadly speaking, ``d.strftime(fmt)`` -acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` -although not all objects support a :meth:`timetuple` method. +control of an explicit format string. Conversely, the :meth:`datetime.strptime` class method creates a :class:`.datetime` object from a string representing a date and time and a -corresponding format string. ``datetime.strptime(date_string, format)`` is -equivalent to ``datetime(*(time.strptime(date_string, format)[0:6]))``, except -when the format includes sub-second components or timezone offset information, -which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``. +corresponding format string. -For :class:`.time` objects, the format codes for year, month, and day should not -be used, as time objects have no such values. If they're used anyway, ``1900`` -is substituted for the year, and ``1`` for the month and day. +The table below provides a high-level comparison of :meth:`strftime` +versus :meth:`strptime`: -For :class:`date` objects, the format codes for hours, minutes, seconds, and -microseconds should not be used, as :class:`date` objects have no such -values. If they're used anyway, ``0`` is substituted for them. ++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ +| | ``strftime`` | ``strptime`` | ++================+========================================================+==============================================================================+ +| Usage | Convert object to a string according to a given format | Parse a string into a :class:`.datetime` object given a corresponding format | ++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ +| Type of method | Instance method | Class method | ++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ +| Method of | :class:`date`; :class:`.datetime`; :class:`.time` | :class:`.datetime` | ++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ +| Signature | ``strftime(format)`` | ``strptime(date_string, format)`` | ++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -The full set of format codes supported varies across platforms, because Python -calls the platform C library's :func:`strftime` function, and platform -variations are common. To see the full set of format codes supported on your -platform, consult the :manpage:`strftime(3)` documentation. -For the same reason, handling of format strings containing Unicode code points -that can't be represented in the charset of the current locale is also -platform-dependent. On some platforms such code points are preserved intact in -the output, while on others ``strftime`` may raise :exc:`UnicodeError` or return -an empty string instead. +:meth:`strftime` and :meth:`strptime` Format Codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The following is a list of all the format codes that the C standard (1989 -version) requires, and these work on all platforms with a standard C -implementation. Note that the 1999 version of the C standard added additional -format codes. +The following is a list of all the format codes that the 1989 C standard +requires, and these work on all platforms with a standard C implementation. +-----------+--------------------------------+------------------------+-------+ | Directive | Meaning | Example | Notes | @@ -2064,7 +2319,7 @@ format codes. | | where 0 is Sunday and 6 is | | | | | Saturday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | | +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | @@ -2077,55 +2332,55 @@ format codes. | | || Januar, Februar, ..., | | | | | Dezember (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | +| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | | +| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | | | number. | 2014, ..., 9998, 9999 | | +-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | | +| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | | +| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | | | AM or PM. || am, pm (de_DE) | \(3) | +-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | +| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4) | -| | decimal number. | | | +| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | +| | decimal number. | | \(9) | +-----------+--------------------------------+------------------------+-------+ | ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | | | number, zero-padded on the | 999999 | | | | left. | | | +-----------+--------------------------------+------------------------+-------+ | ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | -| | ±HHMM[SS[.ffffff]] (empty | -0400, +1030, | | +| | ``±HHMM[SS[.ffffff]]`` (empty | -0400, +1030, | | | | string if the object is | +063415, | | | | naive). | -030712.345216 | | +-----------+--------------------------------+------------------------+-------+ -| ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | +| ``%Z`` | Time zone name (empty string | (empty), UTC, GMT | \(6) | | | if the object is naive). | | | +-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Sunday as the first day of | | | +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| | (Sunday as the first day of | | \(9) | | | the week) as a zero padded | | | | | decimal number. All days in a | | | | | new year preceding the first | | | | | Sunday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Monday as the first day of | | | +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| | (Monday as the first day of | | \(9) | | | the week) as a decimal number. | | | | | All days in a new year | | | | | preceding the first Monday | | | @@ -2148,11 +2403,7 @@ format codes. +-----------+--------------------------------+------------------------+-------+ Several additional directives not required by the C89 standard are included for -convenience. These parameters all correspond to ISO 8601 date values. These -may not be available on all platforms when used with the :meth:`strftime` -method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`strptime` with -incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. +convenience. These parameters all correspond to ISO 8601 date values. +-----------+--------------------------------+------------------------+-------+ | Directive | Meaning | Example | Notes | @@ -2165,16 +2416,59 @@ incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. | ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | | | number where 1 is Monday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8) | -| | number with Monday as | | | +| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | +| | number with Monday as | | \(9) | | | the first day of the week. | | | | | Week 01 is the week containing | | | | | Jan 4. | | | +-----------+--------------------------------+------------------------+-------+ +These may not be available on all platforms when used with the :meth:`strftime` +method. The ISO 8601 year and ISO 8601 week directives are not interchangeable +with the year and week number directives above. Calling :meth:`strptime` with +incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. + +The full set of format codes supported varies across platforms, because Python +calls the platform C library's :func:`strftime` function, and platform +variations are common. To see the full set of format codes supported on your +platform, consult the :manpage:`strftime(3)` documentation. + .. versionadded:: 3.6 ``%G``, ``%u`` and ``%V`` were added. +Technical Detail +^^^^^^^^^^^^^^^^ + +Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's +``time.strftime(fmt, d.timetuple())`` although not all objects support a +:meth:`timetuple` method. + +For the :meth:`datetime.strptime` class method, the default value is +``1900-01-01T00:00:00.000``: any components not specified in the format string +will be pulled from the default value. [#]_ + +Using ``datetime.strptime(date_string, format)`` is equivalent to:: + + datetime(*(time.strptime(date_string, format)[0:6])) + +except when the format includes sub-second components or timezone offset +information, which are supported in ``datetime.strptime`` but are discarded by +``time.strptime``. + +For :class:`.time` objects, the format codes for year, month, and day should not +be used, as :class:`time` objects have no such values. If they're used anyway, +``1900`` is substituted for the year, and ``1`` for the month and day. + +For :class:`date` objects, the format codes for hours, minutes, seconds, and +microseconds should not be used, as :class:`date` objects have no such +values. If they're used anyway, ``0`` is substituted for them. + +For the same reason, handling of format strings containing Unicode code points +that can't be represented in the charset of the current locale is also +platform-dependent. On some platforms such code points are preserved intact in +the output, while on others ``strftime`` may raise :exc:`UnicodeError` or return +an empty string instead. + Notes: (1) @@ -2208,7 +2502,7 @@ Notes: (5) When used with the :meth:`strptime` method, the ``%f`` directive - accepts from one to six digits and zero pads on the right. ``%f`` is + accepts from one to six digits and zero pads on the right. ``%f`` is an extension to the set of format characters in the C standard (but implemented separately in datetime objects, and therefore always available). @@ -2221,13 +2515,13 @@ Notes: ``%z`` :meth:`utcoffset` is transformed into a string of the form - ±HHMM[SS[.ffffff]], where HH is a 2-digit string giving the number of UTC - offset hours, MM is a 2-digit string giving the number of UTC offset - minutes, SS is a 2-digit string giving the number of UTC offset - seconds and ffffff is a 6-digit string giving the number of UTC - offset microseconds. The ffffff part is omitted when the offset is a - whole number of seconds and both the ffffff and the SS part is omitted - when the offset is a whole number of minutes. For example, if + ``±HHMM[SS[.ffffff]]``, where ``HH`` is a 2-digit string giving the number + of UTC offset hours, ``MM`` is a 2-digit string giving the number of UTC + offset minutes, ``SS`` is a 2-digit string giving the number of UTC offset + seconds and ``ffffff`` is a 6-digit string giving the number of UTC + offset microseconds. The ``ffffff`` part is omitted when the offset is a + whole number of seconds and both the ``ffffff`` and the ``SS`` part is + omitted when the offset is a whole number of minutes. For example, if :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string ``'-0330'``. @@ -2242,13 +2536,22 @@ Notes: In addition, providing ``'Z'`` is identical to ``'+00:00'``. ``%Z`` - If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty - string. Otherwise ``%Z`` is replaced by the returned value, which must - be a string. + In :meth:`strftime`, ``%Z`` is replaced by an empty string if + :meth:`tzname` returns ``None``; otherwise ``%Z`` is replaced by the + returned value, which must be a string. + + :meth:`strptime` only accepts certain values for ``%Z``: + + 1. any value in ``time.tzname`` for your machine's locale + 2. the hard-coded values ``UTC`` and ``GMT`` + + So someone living in Japan may have ``JST``, ``UTC``, and ``GMT`` as + valid values, but probably not ``EST``. It will raise ``ValueError`` for + invalid values. .. versionchanged:: 3.2 When the ``%z`` directive is provided to the :meth:`strptime` method, an - aware :class:`.datetime` object will be produced. The ``tzinfo`` of the + aware :class:`.datetime` object will be produced. The ``tzinfo`` of the result will be set to a :class:`timezone` instance. (7) @@ -2262,6 +2565,23 @@ Notes: :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not interchangeable. +(9) + When used with the :meth:`strptime` method, the leading zero is optional + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + ``%W``, and ``%V``. Format ``%y`` does require a leading zero. + .. rubric:: Footnotes .. [#] If, that is, we ignore the effects of Relativity + +.. [#] This matches the definition of the "proleptic Gregorian" calendar in + Dershowitz and Reingold's book *Calendrical Calculations*, + where it's the base calendar for all computations. See the book for + algorithms for converting between proleptic Gregorian ordinals and + many other calendar systems. + +.. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar + `_ + for a good explanation. + +.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year. diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst index 88a2fa62a56588..60223657a44043 100644 --- a/Doc/library/debug.rst +++ b/Doc/library/debug.rst @@ -5,10 +5,13 @@ Debugging and Profiling These libraries help you with Python development: the debugger enables you to step through code, analyze stack frames and set breakpoints etc., and the profilers run code and give you a detailed breakdown of execution times, -allowing you to identify bottlenecks in your programs. +allowing you to identify bottlenecks in your programs. Auditing events +provide visibility into runtime behaviors that would otherwise require +intrusive debugging or patching. .. toctree:: + audit_events.rst bdb.rst faulthandler.rst pdb.rst diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index bcae55eb821784..e194649e30d85c 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1357,6 +1357,9 @@ In addition to the three supplied contexts, new contexts can be created with the The rounding mode of the context is used. Results are always correctly-rounded in the Python version. + ``Decimal(0) ** Decimal(0)`` results in ``InvalidOperation``, and if ``InvalidOperation`` + is not trapped, then results in ``Decimal('NaN')``. + .. versionchanged:: 3.3 The C module computes :meth:`power` in terms of the correctly-rounded :meth:`exp` and :meth:`ln` functions. The result is well-defined but @@ -1475,9 +1478,18 @@ are also included in the pure Python version for compatibility. .. data:: HAVE_THREADS - The default value is ``True``. If Python is compiled without threads, the - C version automatically disables the expensive thread local context - machinery. In this case, the value is ``False``. + The value is ``True``. Deprecated, because Python now always has threads. + +.. deprecated:: 3.9 + +.. data:: HAVE_CONTEXTVAR + + The default value is ``True``. If Python is compiled ``--without-decimal-contextvar``, + the C version uses a thread-local rather than a coroutine-local context and the value + is ``False``. This is slightly faster in some nested context scenarios. + +.. versionadded:: 3.9 backported to 3.7 and 3.8. + Rounding modes -------------- @@ -2121,17 +2133,66 @@ Q. Is the CPython implementation fast for large numbers? A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for -arbitrary precision correctly-rounded decimal floating point arithmetic. +arbitrary precision correctly-rounded decimal floating point arithmetic [#]_. ``libmpdec`` uses `Karatsuba multiplication `_ for medium-sized numbers and the `Number Theoretic Transform `_ -for very large numbers. However, to realize this performance gain, the -context needs to be set for unrounded calculations. +for very large numbers. + +The context must be adapted for exact arbitrary precision arithmetic. :attr:`Emin` +and :attr:`Emax` should always be set to the maximum values, :attr:`clamp` +should always be 0 (the default). Setting :attr:`prec` requires some care. + +The easiest approach for trying out bignum arithmetic is to use the maximum +value for :attr:`prec` as well [#]_:: + + >>> setcontext(Context(prec=MAX_PREC, Emax=MAX_EMAX, Emin=MIN_EMIN)) + >>> x = Decimal(2) ** 256 + >>> x / 128 + Decimal('904625697166532776746648320380374280103671755200316906558262375061821325312') + + +For inexact results, :attr:`MAX_PREC` is far too large on 64-bit platforms and +the available memory will be insufficient:: + + >>> Decimal(1) / 3 + Traceback (most recent call last): + File "", line 1, in + MemoryError + +On systems with overallocation (e.g. Linux), a more sophisticated approach is to +adjust :attr:`prec` to the amount of available RAM. Suppose that you have 8GB of +RAM and expect 10 simultaneous operands using a maximum of 500MB each:: + + >>> import sys + >>> + >>> # Maximum number of digits for a single operand using 500MB in 8-byte words + >>> # with 19 digits per word (4-byte and 9 digits for the 32-bit build): + >>> maxdigits = 19 * ((500 * 1024**2) // 8) + >>> + >>> # Check that this works: + >>> c = Context(prec=maxdigits, Emax=MAX_EMAX, Emin=MIN_EMIN) + >>> c.traps[Inexact] = True + >>> setcontext(c) + >>> + >>> # Fill the available precision with nines: + >>> x = Decimal(0).logical_invert() * 9 + >>> sys.getsizeof(x) + 524288112 + >>> x + 2 + Traceback (most recent call last): + File "", line 1, in + decimal.Inexact: [] + +In general (and especially on systems without overallocation), it is recommended +to estimate even tighter bounds and set the :attr:`Inexact` trap if all calculations +are expected to be exact. + - >>> c = getcontext() - >>> c.prec = MAX_PREC - >>> c.Emax = MAX_EMAX - >>> c.Emin = MIN_EMIN +.. [#] + .. versionadded:: 3.3 -.. versionadded:: 3.3 \ No newline at end of file +.. [#] + .. versionchanged:: 3.9 + This approach now works for all exact results except for non-integer powers. diff --git a/Doc/library/development.rst b/Doc/library/development.rst index ab34e1f7ce5f64..9edce758688e2d 100644 --- a/Doc/library/development.rst +++ b/Doc/library/development.rst @@ -18,12 +18,10 @@ The list of modules described in this chapter is: typing.rst pydoc.rst + devmode.rst doctest.rst unittest.rst unittest.mock.rst unittest.mock-examples.rst 2to3.rst test.rst - -See also the Python development mode: the :option:`-X` ``dev`` option and -:envvar:`PYTHONDEVMODE` environment variable. diff --git a/Doc/library/devmode.rst b/Doc/library/devmode.rst new file mode 100644 index 00000000000000..d5a40cdeeac5ef --- /dev/null +++ b/Doc/library/devmode.rst @@ -0,0 +1,214 @@ +.. _devmode: + +Python Development Mode +======================= + +.. versionadded:: 3.7 + +The Python Development Mode introduces additional runtime checks that are too +expensive to be enabled by default. It should not be more verbose than the +default if the code is correct; new warnings are only emitted when an issue is +detected. + +It can be enabled using the :option:`-X dev <-X>` command line option or by +setting the :envvar:`PYTHONDEVMODE` environment variable to ``1``. + +Effects of the Python Development Mode +====================================== + +Enabling the Python Development Mode is similar to the following command, but +with additional effects described below:: + + PYTHONMALLOC=debug PYTHONASYNCIODEBUG=1 python3 -W default -X faulthandler + +Effects of the Python Development Mode: + +* Add ``default`` :ref:`warning filter `. The + following warnings are shown: + + * :exc:`DeprecationWarning` + * :exc:`ImportWarning` + * :exc:`PendingDeprecationWarning` + * :exc:`ResourceWarning` + + Normally, the above warnings are filtered by the default :ref:`warning + filters `. + + It behaves as if the :option:`-W default <-W>` command line option is used. + + Use the :option:`-W error <-W>` command line option or set the + :envvar:`PYTHONWARNINGS` environment variable to ``error`` to treat warnings + as errors. + +* Install debug hooks on memory allocators to check for: + + * Buffer underflow + * Buffer overflow + * Memory allocator API violation + * Unsafe usage of the GIL + + See the :c:func:`PyMem_SetupDebugHooks` C function. + + It behaves as if the :envvar:`PYTHONMALLOC` environment variable is set to + ``debug``. + + To enable the Python Development Mode without installing debug hooks on + memory allocators, set the :envvar:`PYTHONMALLOC` environment variable to + ``default``. + +* Call :func:`faulthandler.enable` at Python startup to install handlers for + the :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and + :const:`SIGILL` signals to dump the Python traceback on a crash. + + It behaves as if the :option:`-X faulthandler <-X>` command line option is + used or if the :envvar:`PYTHONFAULTHANDLER` environment variable is set to + ``1``. + +* Enable :ref:`asyncio debug mode `. For example, + :mod:`asyncio` checks for coroutines that were not awaited and logs them. + + It behaves as if the :envvar:`PYTHONASYNCIODEBUG` environment variable is set + to ``1``. + +* Check the *encoding* and *errors* arguments for string encoding and decoding + operations. Examples: :func:`open`, :meth:`str.encode` and + :meth:`bytes.decode`. + + By default, for best performance, the *errors* argument is only checked at + the first encoding/decoding error and the *encoding* argument is sometimes + ignored for empty strings. + +* The :class:`io.IOBase` destructor logs ``close()`` exceptions. +* Set the :attr:`~sys.flags.dev_mode` attribute of :attr:`sys.flags` to + ``True``. + +The Python Development Mode does not enable the :mod:`tracemalloc` module by +default, because the overhead cost (to performance and memory) would be too +large. Enabling the :mod:`tracemalloc` module provides additional information +on the origin of some errors. For example, :exc:`ResourceWarning` logs the +traceback where the resource was allocated, and a buffer overflow error logs +the traceback where the memory block was allocated. + +The Python Development Mode does not prevent the :option:`-O` command line +option from removing :keyword:`assert` statements nor from setting +:const:`__debug__` to ``False``. + +.. versionchanged:: 3.8 + The :class:`io.IOBase` destructor now logs ``close()`` exceptions. + +.. versionchanged:: 3.9 + The *encoding* and *errors* arguments are now checked for string encoding + and decoding operations. + + +ResourceWarning Example +======================= + +Example of a script counting the number of lines of the text file specified in +the command line:: + + import sys + + def main(): + fp = open(sys.argv[1]) + nlines = len(fp.readlines()) + print(nlines) + # The file is closed implicitly + + if __name__ == "__main__": + main() + +The script does not close the file explicitly. By default, Python does not emit +any warning. Example using README.txt, which has 269 lines: + +.. code-block:: shell-session + + $ python3 script.py README.txt + 269 + +Enabling the Python Development Mode displays a :exc:`ResourceWarning` warning: + +.. code-block:: shell-session + + $ python3 -X dev script.py README.txt + 269 + script.py:10: ResourceWarning: unclosed file <_io.TextIOWrapper name='README.rst' mode='r' encoding='UTF-8'> + main() + ResourceWarning: Enable tracemalloc to get the object allocation traceback + +In addition, enabling :mod:`tracemalloc` shows the line where the file was +opened: + +.. code-block:: shell-session + + $ python3 -X dev -X tracemalloc=5 script.py README.rst + 269 + script.py:10: ResourceWarning: unclosed file <_io.TextIOWrapper name='README.rst' mode='r' encoding='UTF-8'> + main() + Object allocated at (most recent call last): + File "script.py", lineno 10 + main() + File "script.py", lineno 4 + fp = open(sys.argv[1]) + +The fix is to close explicitly the file. Example using a context manager:: + + def main(): + # Close the file explicitly when exiting the with block + with open(sys.argv[1]) as fp: + nlines = len(fp.readlines()) + print(nlines) + +Not closing a resource explicitly can leave a resource open for way longer than +expected; it can cause severe issues upon exiting Python. It is bad in +CPython, but it is even worse in PyPy. Closing resources explicitly makes an +application more deterministic and more reliable. + + +Bad file descriptor error example +================================= + +Script displaying the first line of itself:: + + import os + + def main(): + fp = open(__file__) + firstline = fp.readline() + print(firstline.rstrip()) + os.close(fp.fileno()) + # The file is closed implicitly + + main() + +By default, Python does not emit any warning: + +.. code-block:: shell-session + + $ python3 script.py + import os + +The Python Development Mode shows a :exc:`ResourceWarning` and logs a "Bad file +descriptor" error when finalizing the file object: + +.. code-block:: shell-session + + $ python3 script.py + import os + script.py:10: ResourceWarning: unclosed file <_io.TextIOWrapper name='script.py' mode='r' encoding='UTF-8'> + main() + ResourceWarning: Enable tracemalloc to get the object allocation traceback + Exception ignored in: <_io.TextIOWrapper name='script.py' mode='r' encoding='UTF-8'> + Traceback (most recent call last): + File "script.py", line 10, in + main() + OSError: [Errno 9] Bad file descriptor + +``os.close(fp.fileno())`` closes the file descriptor. When the file object +finalizer tries to close the file descriptor again, it fails with the ``Bad +file descriptor`` error. A file descriptor must be closed only once. In the +worst case scenario, closing it twice can lead to a crash (see :issue:`18748` +for an example). + +The fix is to remove the ``os.close(fp.fileno())`` line, or open the file with +``closefd=False``. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst new file mode 100644 index 00000000000000..dc82a974ce095d --- /dev/null +++ b/Doc/library/dialog.rst @@ -0,0 +1,230 @@ +Tkinter Dialogs +=============== + +:mod:`tkinter.simpledialog` --- Standard Tkinter input dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. module:: tkinter.simpledialog + :platform: Tk + :synopsis: Simple dialog windows + +**Source code:** :source:`Lib/tkinter/simpledialog.py` + +-------------- + +The :mod:`tkinter.simpledialog` module contains convenience classes and +functions for creating simple modal dialogs to get a value from the user. + + +.. function:: askfloat(title, prompt, **kw) + askinteger(title, prompt, **kw) + askstring(title, prompt, **kw) + + The above three functions provide dialogs that prompt the user to enter a value + of the desired type. + +.. class:: Dialog(parent, title=None) + + The base class for custom dialogs. + + .. method:: body(master) + + Override to construct the dialog's interface and return the widget that + should have initial focus. + + .. method:: buttonbox() + + Default behaviour adds OK and Cancel buttons. Override for custom button + layouts. + + + +:mod:`tkinter.filedialog` --- File selection dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. module:: tkinter.filedialog + :platform: Tk + :synopsis: Dialog classes for file selection + +**Source code:** :source:`Lib/tkinter/filedialog.py` + +-------------- + +The :mod:`tkinter.filedialog` module provides classes and factory functions for +creating file/directory selection windows. + +Native Load/Save Dialogs +------------------------ + +The following classes and functions provide file dialog windows that combine a +native look-and-feel with configuration options to customize behaviour. +The following keyword arguments are applicable to the classes and functions +listed below: + + | *parent* - the window to place the dialog on top of + + | *title* - the title of the window + + | *initialdir* - the directory that the dialog starts in + + | *initialfile* - the file selected upon opening of the dialog + + | *filetypes* - a sequence of (label, pattern) tuples, '*' wildcard is allowed + + | *defaultextension* - default extension to append to file (save dialogs) + + | *multiple* - when true, selection of multiple items is allowed + + +**Static factory functions** + +The below functions when called create a modal, native look-and-feel dialog, +wait for the user's selection, then return the selected value(s) or ``None`` to the +caller. + +.. function:: askopenfile(mode="r", **options) + askopenfiles(mode="r", **options) + + The above two functions create an :class:`Open` dialog and return the opened + file object(s) in read-only mode. + +.. function:: asksaveasfile(mode="w", **options) + + Create a :class:`SaveAs` dialog and return a file object opened in write-only mode. + +.. function:: askopenfilename(**options) + askopenfilenames(**options) + + The above two functions create an :class:`Open` dialog and return the + selected filename(s) that correspond to existing file(s). + +.. function:: asksaveasfilename(**options) + + Create a :class:`SaveAs` dialog and return the selected filename. + +.. function:: askdirectory(**options) + + | Prompt user to select a directory. + | Additional keyword option: + | *mustexist* - determines if selection must be an existing directory. + +.. class:: Open(master=None, **options) + SaveAs(master=None, **options) + + The above two classes provide native dialog windows for saving and loading + files. + +**Convenience classes** + +The below classes are used for creating file/directory windows from scratch. +These do not emulate the native look-and-feel of the platform. + +.. class:: Directory(master=None, **options) + + Create a dialog prompting the user to select a directory. + +.. note:: The *FileDialog* class should be subclassed for custom event + handling and behaviour. + +.. class:: FileDialog(master, title=None) + + Create a basic file selection dialog. + + .. method:: cancel_command(event=None) + + Trigger the termination of the dialog window. + + .. method:: dirs_double_event(event) + + Event handler for double-click event on directory. + + .. method:: dirs_select_event(event) + + Event handler for click event on directory. + + .. method:: files_double_event(event) + + Event handler for double-click event on file. + + .. method:: files_select_event(event) + + Event handler for single-click event on file. + + .. method:: filter_command(event=None) + + Filter the files by directory. + + .. method:: get_filter() + + Retrieve the file filter currently in use. + + .. method:: get_selection() + + Retrieve the currently selected item. + + .. method:: go(dir_or_file=os.curdir, pattern="*", default="", key=None) + + Render dialog and start event loop. + + .. method:: ok_event(event) + + Exit dialog returning current selection. + + .. method:: quit(how=None) + + Exit dialog returning filename, if any. + + .. method:: set_filter(dir, pat) + + Set the file filter. + + .. method:: set_selection(file) + + Update the current file selection to *file*. + + +.. class:: LoadFileDialog(master, title=None) + + A subclass of FileDialog that creates a dialog window for selecting an + existing file. + + .. method:: ok_command() + + Test that a file is provided and that the selection indicates an + already existing file. + +.. class:: SaveFileDialog(master, title=None) + + A subclass of FileDialog that creates a dialog window for selecting a + destination file. + + .. method:: ok_command() + + Test whether or not the selection points to a valid file that is not a + directory. Confirmation is required if an already existing file is + selected. + +:mod:`tkinter.commondialog` --- Dialog window templates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. module:: tkinter.commondialog + :platform: Tk + :synopsis: Tkinter base class for dialogs + +**Source code:** :source:`Lib/tkinter/commondialog.py` + +-------------- + +The :mod:`tkinter.commondialog` module provides the :class:`Dialog` class that +is the base class for dialogs defined in other supporting modules. + +.. class:: Dialog(master=None, **options) + + .. method:: show(color=None, **options) + + Render the Dialog window. + + +.. seealso:: + + Modules :mod:`tkinter.messagebox`, :ref:`tut-files` diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index f044cb2d6e0a86..009b7976dff15f 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -18,12 +18,13 @@ -------------- This module provides classes and functions for comparing sequences. It -can be used for example, for comparing files, and can produce difference -information in various formats, including HTML and context and unified +can be used for example, for comparing files, and can produce information +about file differences in various formats, including HTML and context and unified diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. class:: SequenceMatcher + :noindex: This is a flexible class for comparing pairs of sequences of any type, so long as the sequence elements are :term:`hashable`. The basic algorithm predates, and is a @@ -127,6 +128,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. the next difference highlight at the top of the browser without any leading context). + .. note:: + *fromdesc* and *todesc* are interpreted as unescaped HTML and should be + properly escaped while receiving input from untrusted sources. + .. versionchanged:: 3.5 *charset* keyword-only argument was added. The default charset of HTML document changed from ``'ISO-8859-1'`` to ``'utf-8'``. @@ -334,14 +339,14 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. function:: IS_LINE_JUNK(line) - Return true for ignorable lines. The line *line* is ignorable if *line* is + Return ``True`` for ignorable lines. The line *line* is ignorable if *line* is blank or contains a single ``'#'``, otherwise it is not ignorable. Used as a default for parameter *linejunk* in :func:`ndiff` in older versions. .. function:: IS_CHARACTER_JUNK(ch) - Return true for ignorable characters. The character *ch* is ignorable if *ch* + Return ``True`` for ignorable characters. The character *ch* is ignorable if *ch* is a space or tab, otherwise it is not ignorable. Used as a default for parameter *charjunk* in :func:`ndiff`. @@ -366,7 +371,7 @@ The :class:`SequenceMatcher` class has this constructor: Optional argument *isjunk* must be ``None`` (the default) or a one-argument function that takes a sequence element and returns true if and only if the element is "junk" and should be ignored. Passing ``None`` for *isjunk* is - equivalent to passing ``lambda x: 0``; in other words, no elements are ignored. + equivalent to passing ``lambda x: False``; in other words, no elements are ignored. For example, pass:: lambda x: x in " \t" @@ -417,7 +422,7 @@ The :class:`SequenceMatcher` class has this constructor: is not changed. - .. method:: find_longest_match(alo, ahi, blo, bhi) + .. method:: find_longest_match(alo=0, ahi=None, blo=0, bhi=None) Find longest matching block in ``a[alo:ahi]`` and ``b[blo:bhi]``. @@ -454,6 +459,9 @@ The :class:`SequenceMatcher` class has this constructor: This method returns a :term:`named tuple` ``Match(a, b, size)``. + .. versionchanged:: 3.9 + Added default arguments. + .. method:: get_matching_blocks() @@ -543,6 +551,16 @@ The :class:`SequenceMatcher` class has this constructor: to try :meth:`quick_ratio` or :meth:`real_quick_ratio` first to get an upper bound. + .. note:: + + Caution: The result of a :meth:`ratio` call may depend on the order of + the arguments. For instance:: + + >>> SequenceMatcher(None, 'tide', 'diet').ratio() + 0.25 + >>> SequenceMatcher(None, 'diet', 'tide').ratio() + 0.5 + .. method:: quick_ratio() @@ -615,7 +633,7 @@ If you want to know how to change the first sequence into the second, use work. * `Simple version control recipe - `_ for a small application + `_ for a small application built with :class:`SequenceMatcher`. @@ -634,6 +652,7 @@ The :class:`Differ` class has this constructor: .. class:: Differ(linejunk=None, charjunk=None) + :noindex: Optional keyword parameters *linejunk* and *charjunk* are for filter functions (or ``None``): diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 060d4bb6997a4d..d0307bd8e9f3d0 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -244,7 +244,7 @@ operation is being performed, so the intermediate analysis object isn't useful: .. function:: findlabels(code) - Detect all offsets in the code object *code* which are jump targets, and + Detect all offsets in the raw compiled bytecode string *code* which are jump targets, and return a list of these offsets. @@ -640,15 +640,17 @@ the original TOS1. .. opcode:: LIST_APPEND (i) - Calls ``list.append(TOS[-i], TOS)``. Used to implement list comprehensions. + Calls ``list.append(TOS1[-i], TOS)``. Used to implement list comprehensions. .. opcode:: MAP_ADD (i) - Calls ``dict.setitem(TOS1[-i], TOS, TOS1)``. Used to implement dict + Calls ``dict.__setitem__(TOS1[-i], TOS1, TOS)``. Used to implement dict comprehensions. .. versionadded:: 3.1 + .. versionchanged:: 3.8 + Map value is TOS and map key is TOS1. Before, those were reversed. For all of the :opcode:`SET_ADD`, :opcode:`LIST_APPEND` and :opcode:`MAP_ADD` instructions, while the added value or key/value pair is popped off, the @@ -704,50 +706,29 @@ iterations of the loop. popped values are used to restore the exception state. -.. opcode:: POP_FINALLY (preserve_tos) - - Cleans up the value stack and the block stack. If *preserve_tos* is not - ``0`` TOS first is popped from the stack and pushed on the stack after - perfoming other stack operations: - - * If TOS is ``NULL`` or an integer (pushed by :opcode:`BEGIN_FINALLY` - or :opcode:`CALL_FINALLY`) it is popped from the stack. - * If TOS is an exception type (pushed when an exception has been raised) - 6 values are popped from the stack, the last three popped values are - used to restore the exception state. An exception handler block is - removed from the block stack. +.. opcode:: RERAISE - It is similar to :opcode:`END_FINALLY`, but doesn't change the bytecode - counter nor raise an exception. Used for implementing :keyword:`break`, - :keyword:`continue` and :keyword:`return` in the :keyword:`finally` block. + Re-raises the exception currently on top of the stack. - .. versionadded:: 3.8 + .. versionadded:: 3.9 -.. opcode:: BEGIN_FINALLY +.. opcode:: WITH_EXCEPT_START - Pushes ``NULL`` onto the stack for using it in :opcode:`END_FINALLY`, - :opcode:`POP_FINALLY`, :opcode:`WITH_CLEANUP_START` and - :opcode:`WITH_CLEANUP_FINISH`. Starts the :keyword:`finally` block. + Calls the function in position 7 on the stack with the top three + items on the stack as arguments. + Used to implement the call ``context_manager.__exit__(*exc_info())`` when an exception + has occurred in a :keyword:`with` statement. - .. versionadded:: 3.8 + .. versionadded:: 3.9 -.. opcode:: END_FINALLY +.. opcode:: LOAD_ASSERTION_ERROR - Terminates a :keyword:`finally` clause. The interpreter recalls whether the - exception has to be re-raised or execution has to be continued depending on - the value of TOS. + Pushes :exc:`AssertionError` onto the stack. Used by the :keyword:`assert` + statement. - * If TOS is ``NULL`` (pushed by :opcode:`BEGIN_FINALLY`) continue from - the next instruction. TOS is popped. - * If TOS is an integer (pushed by :opcode:`CALL_FINALLY`), sets the - bytecode counter to TOS. TOS is popped. - * If TOS is an exception type (pushed when an exception has been raised) - 6 values are popped from the stack, the first three popped values are - used to re-raise the exception and the last three popped values are used - to restore the exception state. An exception handler block is removed - from the block stack. + .. versionadded:: 3.9 .. opcode:: LOAD_BUILD_CLASS @@ -770,35 +751,6 @@ iterations of the loop. .. versionadded:: 3.2 -.. opcode:: WITH_CLEANUP_START - - Starts cleaning up the stack when a :keyword:`with` statement block exits. - - At the top of the stack are either ``NULL`` (pushed by - :opcode:`BEGIN_FINALLY`) or 6 values pushed if an exception has been - raised in the with block. Below is the context manager's - :meth:`~object.__exit__` or :meth:`~object.__aexit__` bound method. - - If TOS is ``NULL``, calls ``SECOND(None, None, None)``, - removes the function from the stack, leaving TOS, and pushes ``None`` - to the stack. Otherwise calls ``SEVENTH(TOP, SECOND, THIRD)``, - shifts the bottom 3 values of the stack down, replaces the empty spot - with ``NULL`` and pushes TOS. Finally pushes the result of the call. - - -.. opcode:: WITH_CLEANUP_FINISH - - Finishes cleaning up the stack when a :keyword:`with` statement block exits. - - TOS is result of ``__exit__()`` or ``__aexit__()`` function call pushed - by :opcode:`WITH_CLEANUP_START`. SECOND is ``None`` or an exception type - (pushed when an exception has been raised). - - Pops two values from the stack. If SECOND is not None and TOS is true - unwinds the EXCEPT_HANDLER block which was created when the exception - was caught and pushes ``NULL`` to the stack. - - All of the following opcodes use their arguments. .. opcode:: STORE_NAME (namei) @@ -892,9 +844,9 @@ All of the following opcodes use their arguments. .. opcode:: BUILD_CONST_KEY_MAP (count) - The version of :opcode:`BUILD_MAP` specialized for constant keys. *count* - values are consumed from the stack. The top element on the stack contains - a tuple of keys. + The version of :opcode:`BUILD_MAP` specialized for constant keys. Pops the + top element on the stack which contains a tuple of keys, then starting from + ``TOS1``, pops *count* values to form values in the built dictionary. .. versionadded:: 3.6 @@ -907,61 +859,39 @@ All of the following opcodes use their arguments. .. versionadded:: 3.6 -.. opcode:: BUILD_TUPLE_UNPACK (count) +.. opcode:: LIST_TO_TUPLE - Pops *count* iterables from the stack, joins them in a single tuple, - and pushes the result. Implements iterable unpacking in tuple - displays ``(*x, *y, *z)``. + Pops a list from the stack and pushes a tuple containing the same values. - .. versionadded:: 3.5 + .. versionadded:: 3.9 -.. opcode:: BUILD_TUPLE_UNPACK_WITH_CALL (count) +.. opcode:: LIST_EXTEND (i) - This is similar to :opcode:`BUILD_TUPLE_UNPACK`, - but is used for ``f(*x, *y, *z)`` call syntax. The stack item at position - ``count + 1`` should be the corresponding callable ``f``. + Calls ``list.extend(TOS1[-i], TOS)``. Used to build lists. - .. versionadded:: 3.6 + .. versionadded:: 3.9 -.. opcode:: BUILD_LIST_UNPACK (count) +.. opcode:: SET_UPDATE (i) - This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list - instead of tuple. Implements iterable unpacking in list - displays ``[*x, *y, *z]``. - - .. versionadded:: 3.5 + Calls ``set.update(TOS1[-i], TOS)``. Used to build sets. + .. versionadded:: 3.9 -.. opcode:: BUILD_SET_UNPACK (count) - This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a set - instead of tuple. Implements iterable unpacking in set - displays ``{*x, *y, *z}``. +.. opcode:: DICT_UPDATE (i) - .. versionadded:: 3.5 + Calls ``dict.update(TOS1[-i], TOS)``. Used to build dicts. + .. versionadded:: 3.9 -.. opcode:: BUILD_MAP_UNPACK (count) - Pops *count* mappings from the stack, merges them into a single dictionary, - and pushes the result. Implements dictionary unpacking in dictionary - displays ``{**x, **y, **z}``. - - .. versionadded:: 3.5 +.. opcode:: DICT_MERGE + Like :opcode:`DICT_UPDATE` but raises an exception for duplicate keys. -.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (count) - - This is similar to :opcode:`BUILD_MAP_UNPACK`, - but is used for ``f(**x, **y, **z)`` call syntax. The stack item at - position ``count + 2`` should be the corresponding callable ``f``. - - .. versionadded:: 3.5 - .. versionchanged:: 3.6 - The position of the callable is determined by adding 2 to the opcode - argument instead of encoding it in the second byte of the argument. + .. versionadded:: 3.9 .. opcode:: LOAD_ATTR (namei) @@ -975,6 +905,20 @@ All of the following opcodes use their arguments. ``cmp_op[opname]``. +.. opcode:: IS_OP (invert) + + Performs ``is`` comparison, or ``is not`` if ``invert`` is 1. + + .. versionadded:: 3.9 + + +.. opcode:: CONTAINS_OP (invert) + + Performs ``in`` comparison, or ``not in`` if ``invert`` is 1. + + .. versionadded:: 3.9 + + .. opcode:: IMPORT_NAME (namei) Imports the module ``co_names[namei]``. TOS and TOS1 are popped and provide @@ -1009,6 +953,13 @@ All of the following opcodes use their arguments. .. versionadded:: 3.1 +.. opcode:: JUMP_IF_NOT_EXC_MATCH (target) + + Tests whether the second value on the stack is an exception matching TOS, + and jumps if it is not. Pops two values from the stack. + + .. versionadded:: 3.9 + .. opcode:: JUMP_IF_TRUE_OR_POP (target) @@ -1035,7 +986,7 @@ All of the following opcodes use their arguments. TOS is an :term:`iterator`. Call its :meth:`~iterator.__next__` method. If this yields a new value, push it on the stack (leaving the iterator below - it). If the iterator indicates it is exhausted TOS is popped, and the byte + it). If the iterator indicates it is exhausted, TOS is popped, and the byte code counter is incremented by *delta*. @@ -1050,15 +1001,6 @@ All of the following opcodes use their arguments. stack. *delta* points to the finally block or the first except block. -.. opcode:: CALL_FINALLY (delta) - - Pushes the address of the next instruction onto the stack and increments - bytecode counter by *delta*. Used for calling the finally block as a - "subroutine". - - .. versionadded:: 3.8 - - .. opcode:: LOAD_FAST (var_num) Pushes a reference to the local ``co_varnames[var_num]`` onto the stack. @@ -1113,9 +1055,13 @@ All of the following opcodes use their arguments. .. opcode:: RAISE_VARARGS (argc) - Raises an exception. *argc* indicates the number of arguments to the raise - statement, ranging from 0 to 3. The handler will find the traceback as TOS2, - the parameter as TOS1, and the exception as TOS. + Raises an exception using one of the 3 forms of the ``raise`` statement, + depending on the value of *argc*: + + * 0: ``raise`` (re-raise previous exception) + * 1: ``raise TOS`` (raise exception instance or type at ``TOS``) + * 2: ``raise TOS1 from TOS`` (raise exception instance or type at ``TOS1`` + with ``__cause__`` set to ``TOS``) .. opcode:: CALL_FUNCTION (argc) @@ -1136,8 +1082,10 @@ All of the following opcodes use their arguments. Calls a callable object with positional (if any) and keyword arguments. *argc* indicates the total number of positional and keyword arguments. - The top element on the stack contains a tuple of keyword argument names. - Below that are keyword arguments in the order corresponding to the tuple. + The top element on the stack contains a tuple with the names of the + keyword arguments, which must be strings. + Below that are the values for the keyword arguments, + in the order corresponding to the tuple. Below that are positional arguments, with the right-most parameter on top. Below the arguments is a callable object to call. ``CALL_FUNCTION_KW`` pops all arguments and the callable object off the stack, @@ -1154,10 +1102,6 @@ All of the following opcodes use their arguments. Calls a callable object with variable set of positional and keyword arguments. If the lowest bit of *flags* is set, the top of the stack contains a mapping object containing additional keyword arguments. - Below that is an iterable object containing positional arguments and - a callable object to call. :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and - :opcode:`BUILD_TUPLE_UNPACK_WITH_CALL` can be used for merging multiple - mapping objects and iterables containing arguments. Before the callable is called, the mapping object and iterable object are each "unpacked" and their contents passed in as keyword and positional arguments respectively. @@ -1170,27 +1114,29 @@ All of the following opcodes use their arguments. .. opcode:: LOAD_METHOD (namei) - Loads a method named ``co_names[namei]`` from TOS object. TOS is popped and - method and TOS are pushed when interpreter can call unbound method directly. - TOS will be used as the first argument (``self``) by :opcode:`CALL_METHOD`. - Otherwise, ``NULL`` and method is pushed (method is bound method or - something else). + Loads a method named ``co_names[namei]`` from the TOS object. TOS is popped. + This bytecode distinguishes two cases: if TOS has a method with the correct + name, the bytecode pushes the unbound method and TOS. TOS will be used as + the first argument (``self``) by :opcode:`CALL_METHOD` when calling the + unbound method. Otherwise, ``NULL`` and the object return by the attribute + lookup are pushed. .. versionadded:: 3.7 .. opcode:: CALL_METHOD (argc) - Calls a method. *argc* is number of positional arguments. + Calls a method. *argc* is the number of positional arguments. Keyword arguments are not supported. This opcode is designed to be used with :opcode:`LOAD_METHOD`. Positional arguments are on top of the stack. - Below them, two items described in :opcode:`LOAD_METHOD` on the stack. - All of them are popped and return value is pushed. + Below them, the two items described in :opcode:`LOAD_METHOD` are on the + stack (either ``self`` and an unbound method object or ``NULL`` and an + arbitrary callable). All of them are popped and the return value is pushed. .. versionadded:: 3.7 -.. opcode:: MAKE_FUNCTION (argc) +.. opcode:: MAKE_FUNCTION (flags) Pushes a new function object on the stack. From bottom to top, the consumed stack must consist of values if the argument carries a specified flag value @@ -1215,10 +1161,10 @@ All of the following opcodes use their arguments. .. opcode:: EXTENDED_ARG (ext) - Prefixes any opcode which has an argument too big to fit into the default two - bytes. *ext* holds two additional bytes which, taken together with the - subsequent opcode's argument, comprise a four-byte argument, *ext* being the - two most-significant bytes. + Prefixes any opcode which has an argument too big to fit into the default one + byte. *ext* holds an additional byte which act as higher bits in the argument. + For each opcode, at most three prefixal ``EXTENDED_ARG`` are allowed, forming + an argument from two-byte to four-byte. .. opcode:: FORMAT_VALUE (flags) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index e7c0033eb6bc45..a77322f83acbde 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1,5 +1,3 @@ -:keepdoctest: - :mod:`doctest` --- Test interactive Python examples =================================================== @@ -1531,7 +1529,7 @@ OutputChecker objects A class used to check the whether the actual output from a doctest example matches the expected output. :class:`OutputChecker` defines two methods: - :meth:`check_output`, which compares a given pair of outputs, and returns true + :meth:`check_output`, which compares a given pair of outputs, and returns ``True`` if they match; and :meth:`output_difference`, which returns a string describing the differences between two outputs. diff --git a/Doc/library/dummy_threading.rst b/Doc/library/dummy_threading.rst deleted file mode 100644 index dfc3289abb15c1..00000000000000 --- a/Doc/library/dummy_threading.rst +++ /dev/null @@ -1,20 +0,0 @@ -:mod:`dummy_threading` --- Drop-in replacement for the :mod:`threading` module -============================================================================== - -.. module:: dummy_threading - :synopsis: Drop-in replacement for the threading module. - -**Source code:** :source:`Lib/dummy_threading.py` - -.. deprecated:: 3.7 - Python now always has threading enabled. Please use :mod:`threading` instead. - --------------- - -This module provides a duplicate interface to the :mod:`threading` module. -It was meant to be imported when the :mod:`_thread` module was not provided -on a platform. - -Be careful to not use this module where deadlock might occur from a thread being -created that blocks waiting for another thread to be created. This often occurs -with blocking I/O. diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index f02b35b910a0cf..745b3a6a3ad42a 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -6,6 +6,7 @@ .. module:: email.message :synopsis: The base class representing email messages in a fashion backward compatible with Python 3.2 + :noindex: The :class:`Message` class is very similar to the @@ -307,7 +308,7 @@ Here are the methods of the :class:`Message` class: .. method:: __contains__(name) - Return true if the message object has a field named *name*. Matching is + Return ``True`` if the message object has a field named *name*. Matching is done case-insensitively and *name* should not include the trailing colon. Used for the ``in`` operator, e.g.:: diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 70bf61323c39ed..5d68b104f3a45c 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -12,6 +12,11 @@ This module is part of the legacy (``Compat32``) email API. In the new API the functionality is provided by the *cte* parameter of the :meth:`~email.message.EmailMessage.set_content` method. +This module is deprecated in Python 3. The functions provided here +should not be called explicitly since the :class:`~email.mime.text.MIMEText` +class sets the content type and CTE header using the *_subtype* and *_charset* +values passed during the instantiation of that class. + The remaining text in this section is the original documentation of the module. When creating :class:`~email.message.Message` objects from scratch, you often diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 511ad163583197..f4b9f52509689e 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -99,7 +99,7 @@ All defect classes are subclassed from :class:`email.errors.MessageDefect`. * :class:`MultipartInvariantViolationDefect` -- A message claimed to be a :mimetype:`multipart`, but no subparts were found. Note that when a message has this defect, its :meth:`~email.message.Message.is_multipart` method may - return false even though its content type claims to be :mimetype:`multipart`. + return ``False`` even though its content type claims to be :mimetype:`multipart`. * :class:`InvalidBase64PaddingDefect` -- When decoding a block of base64 encoded bytes, the padding was not correct. Enough padding is added to diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index fc535a3e4399ff..2d9bae6a7ee57b 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -36,7 +36,7 @@ something that contains only ASCII characters, using the standard email RFC Content Transfer Encoding techniques for encoding email messages for transport over channels that are not "8 bit clean". -To accomodate reproducible processing of SMIME-signed messages +To accommodate reproducible processing of SMIME-signed messages :class:`Generator` disables header folding for message parts of type ``multipart/signed`` and all subparts. @@ -188,7 +188,7 @@ to be using :class:`BytesGenerator`, and not :class:`Generator`. (This is required because strings cannot represent non-ASCII bytes.) Convert any bytes with the high bit set as needed using an ASCII-compatible :mailheader:`Content-Transfer-Encoding`. That is, - transform parts with non-ASCII :mailheader:`Cotnent-Transfer-Encoding` + transform parts with non-ASCII :mailheader:`Content-Transfer-Encoding` (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII bytes in headers using the MIME ``unknown-8bit`` character set, thus diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index ce283c6b596cf2..9376da2b8d39ce 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -107,7 +107,7 @@ headers. method if it wishes to set additional attributes beyond those provided by ``BaseHeader`` itself. Such an ``init`` method should look like this:: - def init(self, *args, **kw): + def init(self, /, *args, **kw): self._myattr = kw.pop('myattr') super().init(*args, **kw) @@ -321,19 +321,26 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. The default mappings are: - :subject: UniqueUnstructuredHeader - :date: UniqueDateHeader - :resent-date: DateHeader - :orig-date: UniqueDateHeader - :sender: UniqueSingleAddressHeader - :resent-sender: SingleAddressHeader - :to: UniqueAddressHeader - :resent-to: AddressHeader - :cc: UniqueAddressHeader - :resent-cc: AddressHeader - :from: UniqueAddressHeader - :resent-from: AddressHeader - :reply-to: UniqueAddressHeader + :subject: UniqueUnstructuredHeader + :date: UniqueDateHeader + :resent-date: DateHeader + :orig-date: UniqueDateHeader + :sender: UniqueSingleAddressHeader + :resent-sender: SingleAddressHeader + :to: UniqueAddressHeader + :resent-to: AddressHeader + :cc: UniqueAddressHeader + :resent-cc: AddressHeader + :bcc: UniqueAddressHeader + :resent-bcc: AddressHeader + :from: UniqueAddressHeader + :resent-from: AddressHeader + :reply-to: UniqueAddressHeader + :mime-version: MIMEVersionHeader + :content-type: ContentTypeHeader + :content-disposition: ContentDispositionHeader + :content-transfer-encoding: ContentTransferEncodingHeader + :message-id: MessageIDHeader ``HeaderRegistry`` has the following methods: diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index f1806a0866bb81..5e0509f4181199 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -178,7 +178,7 @@ message objects. .. method:: __contains__(name) - Return true if the message object has a field named *name*. Matching is + Return ``True`` if the message object has a field named *name*. Matching is done without regard to case and *name* does not include the trailing colon. Used for the ``in`` operator. For example:: diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 63fae2ab84e213..4d0e920eb0ad29 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -117,8 +117,8 @@ of the new API. a 10-tuple; the first 9 elements make up a tuple that can be passed directly to :func:`time.mktime`, and the tenth is the offset of the date's timezone from UTC (which is the official term for Greenwich Mean Time) [#]_. If the input string - has no timezone, the last element of the tuple returned is ``None``. Note that - indexes 6, 7, and 8 of the result tuple are not usable. + has no timezone, the last element of the tuple returned is ``0``, which represents + UTC. Note that indexes 6, 7, and 8 of the result tuple are not usable. .. function:: parsedate_to_datetime(date) diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index c797f63326d1a2..a5221250c40486 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -74,7 +74,7 @@ options: script will *not* be installed. * ``--default-pip``: if a "default pip" installation is requested, the - ``pip`` script will be installed in addition to the two regular scripts. + ``pip`` script will be installed in addition to the two regular scripts. Providing both of the script selection options will trigger an exception. @@ -119,6 +119,8 @@ Module API *verbosity* controls the level of output to :data:`sys.stdout` from the bootstrapping operation. + .. audit-event:: ensurepip.bootstrap root ensurepip.bootstrap + .. note:: The bootstrapping process has side effects on both ``sys.path`` and diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 19277d76995fed..2f84be229bc4da 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -19,6 +19,12 @@ An enumeration is a set of symbolic names (members) bound to unique, constant values. Within an enumeration, the members can be compared by identity, and the enumeration itself can be iterated over. +.. note:: Case of Enum Members + + Because Enums are used to represent constants we recommend using + UPPER_CASE names for enum members, and will be using that style + in our examples. + Module Contents --------------- @@ -50,12 +56,13 @@ helper, :class:`auto`. the bitwise operations without losing their :class:`Flag` membership. .. function:: unique + :noindex: Enum class decorator that ensures only one name is bound to any one value. .. class:: auto - Instances are replaced with an appropriate value for Enum members. + Instances are replaced with an appropriate value for Enum members. By default, the initial value starts at 1. .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` @@ -113,7 +120,6 @@ The *type* of an enumeration member is the enumeration it belongs to:: >>> isinstance(Color.GREEN, Color) True - >>> Enum members also have a property that contains just their item name:: @@ -273,6 +279,10 @@ overridden:: the next :class:`int` in sequence with the last :class:`int` provided, but the way it does this is an implementation detail and may change. +.. note:: + + The :meth:`_generate_next_value_` method must be defined before any members. + Iteration --------- @@ -383,8 +393,8 @@ enumeration, with the exception of special methods (:meth:`__str__`, variable names listed in :attr:`_ignore_`. Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then -whatever value(s) were given to the enum member will be passed into those -methods. See `Planet`_ for an example. +any value(s) given to the enum member will be passed into those methods. +See `Planet`_ for an example. Restricted Enum subclassing @@ -646,6 +656,13 @@ be combined with them:: >>> Perm.X | 8 +:class:`IntFlag` members can also be iterated over:: + + >>> list(RW) + [, ] + +.. versionadded:: 3.10 + Flag ^^^^ @@ -699,6 +716,14 @@ value:: >>> bool(Color.BLACK) False +:class:`Flag` members can also be iterated over:: + + >>> purple = Color.RED | Color.BLUE + >>> list(purple) + [, ] + +.. versionadded:: 3.10 + .. note:: For the majority of new code, :class:`Enum` and :class:`Flag` are strongly @@ -730,8 +755,7 @@ Some rules: 2. While :class:`Enum` can have members of any type, once you mix in an additional type, all the members must have values of that type, e.g. :class:`int` above. This restriction does not apply to mix-ins which only - add methods and don't specify another data type such as :class:`int` or - :class:`str`. + add methods and don't specify another type. 3. When another data type is mixed in, the :attr:`value` attribute is *not the same* as the enum member itself, although it is equivalent and will compare equal. @@ -739,9 +763,11 @@ Some rules: :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. 5. :ref:`Formatted string literals `, :meth:`str.format`, - and :func:`format` will use the mixed-in - type's :meth:`__format__`. If the :class:`Enum` class's :func:`str` or - :func:`repr` is desired, use the `!s` or `!r` format codes. + and :func:`format` will use the mixed-in type's :meth:`__format__` + unless :meth:`__str__` or :meth:`__format__` is overridden in the subclass, + in which case the overridden methods or :class:`Enum` methods will be used. + Use the !s and !r format codes to force usage of the :class:`Enum` class's + :meth:`__str__` and :meth:`__repr__` methods. When to use :meth:`__new__` vs. :meth:`__init__` ------------------------------------------------ @@ -1052,7 +1078,7 @@ Supported ``_sunder_`` names - ``_missing_`` -- a lookup function used when a value is not found; may be overridden -- ``_ignore_`` -- a list of names, either as a :func:`list` or a :func:`str`, +- ``_ignore_`` -- a list of names, either as a :class:`list` or a :class:`str`, that will not be transformed into members, and will be removed from the final class - ``_order_`` -- used in Python 2/3 code to ensure member order is consistent diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 52a505e0a0ff8f..df2cda9d67ad15 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -688,6 +688,10 @@ The following exceptions are used as warning categories; see the Base class for warnings about deprecated features when those warnings are intended for other Python developers. + Ignored by the default warning filters, except in the ``__main__`` module + (:pep:`565`). Enabling the :ref:`Python Development Mode ` shows + this warning. + .. exception:: PendingDeprecationWarning @@ -699,6 +703,9 @@ The following exceptions are used as warning categories; see the upcoming deprecation is unusual, and :exc:`DeprecationWarning` is preferred for already active deprecations. + Ignored by the default warning filters. Enabling the :ref:`Python + Development Mode ` shows this warning. + .. exception:: SyntaxWarning @@ -720,6 +727,9 @@ The following exceptions are used as warning categories; see the Base class for warnings about probable mistakes in module imports. + Ignored by the default warning filters. Enabling the :ref:`Python + Development Mode ` shows this warning. + .. exception:: UnicodeWarning @@ -733,8 +743,10 @@ The following exceptions are used as warning categories; see the .. exception:: ResourceWarning - Base class for warnings related to resource usage. Ignored by the default - warning filters. + Base class for warnings related to resource usage. + + Ignored by the default warning filters. Enabling the :ref:`Python + Development Mode ` shows this warning. .. versionadded:: 3.2 diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 94ebd87639c501..59274c1dd7ec35 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -40,6 +40,9 @@ alternatively be passed to :func:`faulthandler.enable`. The module is implemented in C, so tracebacks can be dumped on a crash or when Python is deadlocked. +The :ref:`Python Development Mode ` calls :func:`faulthandler.enable` +at Python startup. + Dumping the traceback --------------------- @@ -100,8 +103,10 @@ Dumping the tracebacks after a timeout :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file descriptors `. - This function is implemented using a watchdog thread and therefore is not - available if Python is compiled with threads disabled. + This function is implemented using a watchdog thread. + + .. versionchanged:: 3.7 + This function is now always available. .. versionchanged:: 3.5 Added support for passing file descriptor to this function. diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 88112f6b7e545b..07a15d27216e92 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -28,6 +28,16 @@ descriptor. Operations in this module used to raise an :exc:`IOError` where they now raise an :exc:`OSError`. +.. versionchanged:: 3.8 + The fcntl module now contains ``F_ADD_SEALS``, ``F_GET_SEALS``, and + ``F_SEAL_*`` constants for sealing of :func:`os.memfd_create` file + descriptors. + +.. versionchanged:: 3.9 + On macOS, the fcntl module exposes the ``F_GETPATH`` constant, which obtains + the path of a file from a file descriptor. + On Linux(>=3.15), the fcntl module exposes the ``F_OFD_GETLK``, ``F_OFD_SETLK`` + and ``F_OFD_SETLKW`` constants, which working with open file description locks. The module defines the following functions: @@ -53,6 +63,8 @@ The module defines the following functions: If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. + .. audit-event:: fcntl.fcntl fd,cmd,arg fcntl.fcntl + .. function:: ioctl(fd, request, arg=0, mutate_flag=True) @@ -102,6 +114,8 @@ The module defines the following functions: >>> buf array('h', [13341]) + .. audit-event:: fcntl.ioctl fd,request,arg fcntl.ioctl + .. function:: flock(fd, operation) @@ -112,11 +126,14 @@ The module defines the following functions: If the :c:func:`flock` fails, an :exc:`OSError` exception is raised. + .. audit-event:: fcntl.flock fd,operation fcntl.flock + .. function:: lockf(fd, cmd, len=0, start=0, whence=0) This is essentially a wrapper around the :func:`~fcntl.fcntl` locking calls. - *fd* is the file descriptor of the file to lock or unlock, and *cmd* + *fd* is the file descriptor (file objects providing a :meth:`~io.IOBase.fileno` + method are accepted as well) of the file to lock or unlock, and *cmd* is one of the following values: * :const:`LOCK_UN` -- unlock @@ -144,6 +161,8 @@ The module defines the following functions: The default for *len* is 0 which means to lock to the end of the file. The default for *whence* is also 0. + .. audit-event:: fcntl.lockf fd,cmd,len,start,whence fcntl.lockf + Examples (all on a SVR4 compliant system):: import struct, fcntl, os @@ -167,4 +186,3 @@ using the :func:`flock` call may be better. present in the :mod:`os` module (on BSD only), the :func:`os.open` function provides an alternative to the :func:`lockf` and :func:`flock` functions. - diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index af9dff34a80827..cc4039a30e38ae 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -23,8 +23,9 @@ The typical use is:: This iterates over the lines of all files listed in ``sys.argv[1:]``, defaulting to ``sys.stdin`` if the list is empty. If a filename is ``'-'``, it is also -replaced by ``sys.stdin``. To specify an alternative list of filenames, pass it -as the first argument to :func:`.input`. A single file name is also allowed. +replaced by ``sys.stdin`` and the optional arguments *mode* and *openhook* +are ignored. To specify an alternative list of filenames, pass it as the +first argument to :func:`.input`. A single file name is also allowed. All files are opened in text mode by default, but you can override this by specifying the *mode* parameter in the call to :func:`.input` or @@ -54,7 +55,7 @@ provided by this module. The following function is the primary interface of this module: -.. function:: input(files=None, inplace=False, backup='', bufsize=0, mode='r', openhook=None) +.. function:: input(files=None, inplace=False, backup='', *, mode='r', openhook=None) Create an instance of the :class:`FileInput` class. The instance will be used as global state for the functions of this module, and is also returned to use @@ -72,8 +73,9 @@ The following function is the primary interface of this module: .. versionchanged:: 3.2 Can be used as a context manager. - .. deprecated-removed:: 3.6 3.8 - The *bufsize* parameter. + .. versionchanged:: 3.8 + The keyword parameters *mode* and *openhook* are now keyword-only. + The following functions use the global state created by :func:`fileinput.input`; if there is no active state, :exc:`RuntimeError` is raised. @@ -107,14 +109,14 @@ if there is no active state, :exc:`RuntimeError` is raised. .. function:: isfirstline() - Returns true if the line just read is the first line of its file, otherwise - returns false. + Return ``True`` if the line just read is the first line of its file, otherwise + return ``False``. .. function:: isstdin() - Returns true if the last line was read from ``sys.stdin``, otherwise returns - false. + Return ``True`` if the last line was read from ``sys.stdin``, otherwise return + ``False``. .. function:: nextfile() @@ -135,7 +137,7 @@ The class which implements the sequence behavior provided by the module is available for subclassing as well: -.. class:: FileInput(files=None, inplace=False, backup='', bufsize=0, mode='r', openhook=None) +.. class:: FileInput(files=None, inplace=False, backup='', *, mode='r', openhook=None) Class :class:`FileInput` is the implementation; its methods :meth:`filename`, :meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`, @@ -160,18 +162,20 @@ available for subclassing as well: with FileInput(files=('spam.txt', 'eggs.txt')) as input: process(input) + .. versionchanged:: 3.2 Can be used as a context manager. .. deprecated:: 3.4 The ``'rU'`` and ``'U'`` modes. - .. deprecated-removed:: 3.6 3.8 - The *bufsize* parameter. - .. deprecated:: 3.8 Support for :meth:`__getitem__` method is deprecated. + .. versionchanged:: 3.8 + The keyword parameter *mode* and *openhook* are now keyword-only. + + **Optional in-place filtering:** if the keyword argument ``inplace=True`` is passed to :func:`fileinput.input` or to the :class:`FileInput` constructor, the diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index b5a818e1cafa61..a4d006eb58ffeb 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -84,6 +84,10 @@ another rational number, or from a string. The :class:`Fraction` constructor now accepts :class:`float` and :class:`decimal.Decimal` instances. + .. versionchanged:: 3.9 + The :func:`math.gcd` function is now used to normalize the *numerator* + and *denominator*. :func:`math.gcd` always return a :class:`int` type. + Previously, the GCD type depended on *numerator* and *denominator*. .. attribute:: numerator @@ -94,6 +98,13 @@ another rational number, or from a string. Denominator of the Fraction in lowest term. + .. method:: as_integer_ratio() + + Return a tuple of two integers, whose ratio is equal + to the Fraction and with a positive denominator. + + .. versionadded:: 3.8 + .. method:: from_float(flt) This class method constructs a :class:`Fraction` representing the exact @@ -165,18 +176,6 @@ another rational number, or from a string. method can also be accessed through the :func:`round` function. -.. function:: gcd(a, b) - - Return the greatest common divisor of the integers *a* and *b*. If either - *a* or *b* is nonzero, then the absolute value of ``gcd(a, b)`` is the - largest integer that divides both *a* and *b*. ``gcd(a,b)`` has the same - sign as *b* if *b* is nonzero; otherwise it takes the sign of *a*. ``gcd(0, - 0)`` returns ``0``. - - .. deprecated:: 3.5 - Use :func:`math.gcd` instead. - - .. seealso:: Module :mod:`numbers` diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 6c39f9a59fc1fd..f4d4cdf9ada9d9 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -19,10 +19,12 @@ as mirroring other FTP servers. It is also used by the module :mod:`urllib.request` to handle URLs that use FTP. For more information on FTP (File Transfer Protocol), see Internet :rfc:`959`. +The default encoding is UTF-8, following :rfc:`2640`. + Here's a sample session using the :mod:`ftplib` module:: >>> from ftplib import FTP - >>> ftp = FTP('ftp.debian.org') # connect to host, default port + >>> ftp = FTP('ftp.us.debian.org') # connect to host, default port >>> ftp.login() # user anonymous, passwd anonymous@ '230 Login successful.' >>> ftp.cwd('debian') # change into "debian" directory @@ -33,14 +35,15 @@ Here's a sample session using the :mod:`ftplib` module:: drwxr-sr-x 4 1176 1176 4096 Nov 17 2008 project drwxr-xr-x 3 1176 1176 4096 Oct 10 2012 tools '226 Directory send OK.' - >>> ftp.retrbinary('RETR README', open('README', 'wb').write) + >>> with open('README', 'wb') as fp: + >>> ftp.retrbinary('RETR README', fp.write) '226 Transfer complete.' >>> ftp.quit() The module defines the following items: -.. class:: FTP(host='', user='', passwd='', acct='', timeout=None, source_address=None) +.. class:: FTP(host='', user='', passwd='', acct='', timeout=None, source_address=None, *, encoding='utf-8') Return a new instance of the :class:`FTP` class. When *host* is given, the method call ``connect(host)`` is made. When *user* is given, additionally @@ -49,7 +52,8 @@ The module defines the following items: parameter specifies a timeout in seconds for blocking operations like the connection attempt (if is not specified, the global default timeout setting will be used). *source_address* is a 2-tuple ``(host, port)`` for the socket - to bind to as its source address before connecting. + to bind to as its source address before connecting. The *encoding* parameter + specifies the encoding for directories and filenames. The :class:`FTP` class supports the :keyword:`with` statement, e.g.: @@ -71,8 +75,13 @@ The module defines the following items: .. versionchanged:: 3.3 *source_address* parameter was added. + .. versionchanged:: 3.9 + If the *timeout* parameter is set to be zero, it will raise a + :class:`ValueError` to prevent the creation of a non-blocking socket. + The *encoding* parameter was added, and the default was changed from + Latin-1 to UTF-8 to follow :rfc:`2640`. -.. class:: FTP_TLS(host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, timeout=None, source_address=None) +.. class:: FTP_TLS(host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, timeout=None, source_address=None, *, encoding='utf-8') A :class:`FTP` subclass which adds TLS support to FTP as described in :rfc:`4217`. @@ -104,6 +113,12 @@ The module defines the following items: :func:`ssl.create_default_context` select the system's trusted CA certificates for you. + .. versionchanged:: 3.9 + If the *timeout* parameter is set to be zero, it will raise a + :class:`ValueError` to prevent the creation of a non-blocking socket. + The *encoding* parameter was added, and the default was changed from + Latin-1 to UTF-8 to follow :rfc:`2640`. + Here's a sample session using the :class:`FTP_TLS` class:: >>> ftps = FTP_TLS('ftp.pureftpd.org') @@ -144,7 +159,7 @@ The module defines the following items: The set of all exceptions (as a tuple) that methods of :class:`FTP` instances may raise as a result of problems with the FTP connection (as opposed to programming errors made by the caller). This set includes the - four exceptions listed above as well as :exc:`OSError`. + four exceptions listed above as well as :exc:`OSError` and :exc:`EOFError`. .. seealso:: @@ -190,6 +205,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as its source address before connecting. + .. audit-event:: ftplib.connect self,host,port ftplib.FTP.connect + .. versionchanged:: 3.3 *source_address* parameter was added. @@ -223,6 +240,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Send a simple command string to the server and return the response string. + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.sendcmd + .. method:: FTP.voidcmd(cmd) @@ -230,6 +249,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. nothing if a response code corresponding to success (codes in the range 200--299) is received. Raise :exc:`error_reply` otherwise. + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd + .. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) @@ -245,9 +266,10 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. .. method:: FTP.retrlines(cmd, callback=None) - Retrieve a file or directory listing in ASCII transfer mode. *cmd* should be - an appropriate ``RETR`` command (see :meth:`retrbinary`) or a command such as - ``LIST`` or ``NLST`` (usually just the string ``'LIST'``). + Retrieve a file or directory listing in the encoding specified by the + *encoding* parameter at initialization. + *cmd* should be an appropriate ``RETR`` command (see :meth:`retrbinary`) or + a command such as ``LIST`` or ``NLST`` (usually just the string ``'LIST'``). ``LIST`` retrieves a list of files and information about those files. ``NLST`` retrieves a list of file names. The *callback* function is called for each line with a string argument @@ -277,7 +299,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. .. method:: FTP.storlines(cmd, fp, callback=None) - Store a file in ASCII transfer mode. *cmd* should be an appropriate + Store a file in line mode. *cmd* should be an appropriate ``STOR`` command (see :meth:`storbinary`). Lines are read until EOF from the :term:`file object` *fp* (opened in binary mode) using its :meth:`~io.IOBase.readline` method to provide the data to be stored. *callback* is an optional single @@ -295,10 +317,9 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. If optional *rest* is given, a ``REST`` command is sent to the server, passing *rest* as an argument. *rest* is usually a byte offset into the requested file, telling the server to restart sending the file's bytes at the requested offset, - skipping over the initial bytes. Note however that :rfc:`959` requires only that - *rest* be a string containing characters in the printable range from ASCII code - 33 to ASCII code 126. The :meth:`transfercmd` method, therefore, converts - *rest* to a string, but no check is performed on the string's contents. If the + skipping over the initial bytes. Note however that the :meth:`transfercmd` + method converts *rest* to a string with the *encoding* parameter specified + at initialization, but no check is performed on the string's contents. If the server does not recognize the ``REST`` command, an :exc:`error_reply` exception will be raised. If this happens, simply call :meth:`transfercmd` without a *rest* argument. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 613e4f74ac4176..7543fc4b10d466 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -7,24 +7,38 @@ Built-in Functions The Python interpreter has a number of functions and types built into it that are always available. They are listed here in alphabetical order. -=================== ================= ================== ================== ==================== -.. .. Built-in Functions .. .. -=================== ================= ================== ================== ==================== -:func:`abs` :func:`delattr` :func:`hash` |func-memoryview|_ |func-set|_ -:func:`all` |func-dict|_ :func:`help` :func:`min` :func:`setattr` -:func:`any` :func:`dir` :func:`hex` :func:`next` :func:`slice` -:func:`ascii` :func:`divmod` :func:`id` :func:`object` :func:`sorted` -:func:`bin` :func:`enumerate` :func:`input` :func:`oct` :func:`staticmethod` -:func:`bool` :func:`eval` :func:`int` :func:`open` |func-str|_ -:func:`breakpoint` :func:`exec` :func:`isinstance` :func:`ord` :func:`sum` -|func-bytearray|_ :func:`filter` :func:`issubclass` :func:`pow` :func:`super` -|func-bytes|_ :func:`float` :func:`iter` :func:`print` |func-tuple|_ -:func:`callable` :func:`format` :func:`len` :func:`property` :func:`type` -:func:`chr` |func-frozenset|_ |func-list|_ |func-range|_ :func:`vars` -:func:`classmethod` :func:`getattr` :func:`locals` :func:`repr` :func:`zip` -:func:`compile` :func:`globals` :func:`map` :func:`reversed` :func:`__import__` -:func:`complex` :func:`hasattr` :func:`max` :func:`round` -=================== ================= ================== ================== ==================== ++---------------------------------------------------------------------------------------------------+ +| Built-in Functions | ++=========================+=======================+=======================+=========================+ +| | **A** | | **E** | | **L** | | **R** | +| | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | +| | :func:`all` | | :func:`eval` | | |func-list|_ | | :func:`repr` | +| | :func:`any` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | +| | :func:`ascii` | | | | | | :func:`round` | +| | | | **F** | | **M** | | | +| | **B** | | :func:`filter` | | :func:`map` | | **S** | +| | :func:`bin` | | :func:`float` | | :func:`max` | | |func-set|_ | +| | :func:`bool` | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | +| | :func:`breakpoint` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | +| | |func-bytearray|_ | | | | | | :func:`sorted` | +| | |func-bytes|_ | | **G** | | **N** | | :func:`staticmethod` | +| | | | :func:`getattr` | | :func:`next` | | |func-str|_ | +| | **C** | | :func:`globals` | | | | :func:`sum` | +| | :func:`callable` | | | | **O** | | :func:`super` | +| | :func:`chr` | | **H** | | :func:`object` | | | +| | :func:`classmethod` | | :func:`hasattr` | | :func:`oct` | | **T** | +| | :func:`compile` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | +| | :func:`complex` | | :func:`help` | | :func:`ord` | | :func:`type` | +| | | | :func:`hex` | | | | | +| | **D** | | | | **P** | | **V** | +| | :func:`delattr` | | **I** | | :func:`pow` | | :func:`vars` | +| | |func-dict|_ | | :func:`id` | | :func:`print` | | | +| | :func:`dir` | | :func:`input` | | :func:`property` | | **Z** | +| | :func:`divmod` | | :func:`int` | | | | :func:`zip` | +| | | | :func:`isinstance` | | | | | +| | | | :func:`issubclass` | | | | **_** | +| | | | :func:`iter` | | | | :func:`__import__` | ++-------------------------+-----------------------+-----------------------+-------------------------+ .. using :func:`dict` would create a link to another page, so local targets are used, with replacement texts to make the output in the table consistent @@ -43,9 +57,8 @@ are always available. They are listed here in alphabetical order. .. function:: abs(x) Return the absolute value of a number. The argument may be an - integer or a floating point number. If the argument is a complex number, its - magnitude is returned. If *x* defines :meth:`__abs__`, - ``abs(x)`` returns ``x.__abs__()``. + integer, a floating point number, or an object implementing :meth:`__abs__`. + If the argument is a complex number, its magnitude is returned. .. function:: all(iterable) @@ -128,6 +141,8 @@ are always available. They are listed here in alphabetical order. :func:`breakpoint` will automatically call that, allowing you to drop into the debugger of choice. + .. audit-event:: builtins.breakpoint breakpointhook breakpoint + .. versionadded:: 3.7 .. _func-bytearray: @@ -179,8 +194,8 @@ are always available. They are listed here in alphabetical order. .. function:: callable(object) Return :const:`True` if the *object* argument appears callable, - :const:`False` if not. If this returns true, it is still possible that a - call fails, but if it is false, calling *object* will never succeed. + :const:`False` if not. If this returns ``True``, it is still possible that a + call fails, but if it is ``False``, calling *object* will never succeed. Note that classes are callable (calling a class returns a new instance); instances are callable if their class has a :meth:`__call__` method. @@ -220,10 +235,12 @@ are always available. They are listed here in alphabetical order. implied first argument. Class methods are different than C++ or Java static methods. If you want those, - see :func:`staticmethod`. - + see :func:`staticmethod` in this section. For more information on class methods, see :ref:`types`. + .. versionchanged:: 3.9 + Class methods can now wrap other :term:`descriptors ` such as + :func:`property`. .. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) @@ -257,6 +274,12 @@ are always available. They are listed here in alphabetical order. can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on the :class:`~__future__._Feature` instance in the :mod:`__future__` module. + The optional argument *flags* also controls whether the compiled source is + allowed to contain top-level ``await``, ``async for`` and ``async with``. + When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the return code + object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively + executed via ``await eval(code_object)``. + The argument *optimize* specifies the optimization level of the compiler; the default value of ``-1`` selects the optimization level of the interpreter as given by :option:`-O` options. Explicit levels are ``0`` (no optimization; @@ -269,6 +292,12 @@ are always available. They are listed here in alphabetical order. If you want to parse Python code into its AST representation, see :func:`ast.parse`. + .. audit-event:: compile source,filename compile + + Raises an :ref:`auditing event ` ``compile`` with arguments + ``source`` and ``filename``. This event may also be raised by implicit + compilation. + .. note:: When compiling a string with multi-line code in ``'single'`` or @@ -290,6 +319,10 @@ are always available. They are listed here in alphabetical order. Previously, :exc:`TypeError` was raised when null bytes were encountered in *source*. + .. versionadded:: 3.8 + ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable + support for top-level ``await``, ``async for``, and ``async with``. + .. class:: complex([real[, imag]]) @@ -302,6 +335,11 @@ are always available. They are listed here in alphabetical order. :class:`int` and :class:`float`. If both arguments are omitted, returns ``0j``. + For a general Python object ``x``, ``complex(x)`` delegates to + ``x.__complex__()``. If ``__complex__()`` is not defined then it falls back + to :meth:`__float__`. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. + .. note:: When converting from a string, the string must not contain whitespace @@ -314,6 +352,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.6 Grouping digits with underscores as in code literals is allowed. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__complex__` and + :meth:`__float__` are not defined. + .. function:: delattr(object, name) @@ -427,7 +469,7 @@ are always available. They are listed here in alphabetical order. n += 1 -.. function:: eval(expression, globals=None, locals=None) +.. function:: eval(expression[, globals[, locals]]) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping @@ -438,12 +480,16 @@ are always available. They are listed here in alphabetical order. dictionaries as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *expression* is parsed. - This means that *expression* normally has full - access to the standard :mod:`builtins` module and restricted environments are - propagated. If the *locals* dictionary is omitted it defaults to the *globals* - dictionary. If both dictionaries are omitted, the expression is executed in the - environment where :func:`eval` is called. The return value is the result of + inserted under that key before *expression* is parsed. This means that + *expression* normally has full access to the standard :mod:`builtins` + module and restricted environments are propagated. If the *locals* + dictionary is omitted it defaults to the *globals* dictionary. If both + dictionaries are omitted, the expression is executed with the *globals* and + *locals* in the environment where :func:`eval` is called. Note, *eval()* + does not have access to the :term:`nested scopes ` (non-locals) in the + enclosing environment. + + The return value is the result of the evaluated expression. Syntax errors are reported as exceptions. Example: >>> x = 1 @@ -463,6 +509,11 @@ are always available. They are listed here in alphabetical order. See :func:`ast.literal_eval` for a function that can safely evaluate strings with expressions containing only literals. + .. audit-event:: exec code_object eval + + Raises an :ref:`auditing event ` ``exec`` with the code object + as the argument. Code compilation events may also be raised. + .. index:: builtin: exec .. function:: exec(object[, globals[, locals]]) @@ -478,7 +529,8 @@ are always available. They are listed here in alphabetical order. :func:`exec` function. The return value is ``None``. In all cases, if the optional parts are omitted, the code is executed in the - current scope. If only *globals* is provided, it must be a dictionary, which + current scope. If only *globals* is provided, it must be a dictionary + (and not a subclass of dictionary), which will be used for both the global and the local variables. If *globals* and *locals* are given, they are used for the global and local variables, respectively. If provided, *locals* can be any mapping object. Remember @@ -492,6 +544,11 @@ are always available. They are listed here in alphabetical order. builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. + .. audit-event:: exec code_object exec + + Raises an :ref:`auditing event ` ``exec`` with the code object + as the argument. Code compilation events may also be raised. + .. note:: The built-in functions :func:`globals` and :func:`locals` return the current @@ -539,7 +596,7 @@ are always available. They are listed here in alphabetical order. input must conform to the following grammar after leading and trailing whitespace characters are removed: - .. productionlist:: + .. productionlist:: float sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" @@ -557,7 +614,8 @@ are always available. They are listed here in alphabetical order. float, an :exc:`OverflowError` will be raised. For a general Python object ``x``, ``float(x)`` delegates to - ``x.__float__()``. + ``x.__float__()``. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. If no argument is given, ``0.0`` is returned. @@ -582,6 +640,9 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.7 *x* is now a positional-only parameter. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__float__` is not defined. + .. index:: single: __format__ @@ -737,13 +798,24 @@ are always available. They are listed here in alphabetical order. If the :mod:`readline` module was loaded, then :func:`input` will use it to provide elaborate line editing and history features. + .. audit-event:: builtins.input prompt input + + Raises an :ref:`auditing event ` ``builtins.input`` with + argument ``prompt`` before reading input + + .. audit-event:: builtins.input/result result input + + Raises an auditing event ``builtins.input/result`` with the result after + successfully reading input. + .. class:: int([x]) int(x, base=10) Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* defines :meth:`__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__trunc__`, + ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__index__`, + it returns ``x.__index__()``. If *x* defines :meth:`__trunc__`, it returns ``x.__trunc__()``. For floating point numbers, this truncates towards zero. @@ -775,22 +847,25 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.7 *x* is now a positional-only parameter. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__int__` is not defined. + .. function:: isinstance(object, classinfo) - Return true if the *object* argument is an instance of the *classinfo* + Return ``True`` if the *object* argument is an instance of the *classinfo* argument, or of a (direct, indirect or :term:`virtual `) subclass thereof. If *object* is not - an object of the given type, the function always returns false. + an object of the given type, the function always returns ``False``. If *classinfo* is a tuple of type objects (or recursively, other such - tuples), return true if *object* is an instance of any of the types. + tuples), return ``True`` if *object* is an instance of any of the types. If *classinfo* is not a type or tuple of types and such tuples, a :exc:`TypeError` exception is raised. .. function:: issubclass(class, classinfo) - Return true if *class* is a subclass (direct, indirect or :term:`virtual + Return ``True`` if *class* is a subclass (direct, indirect or :term:`virtual `) of *classinfo*. A class is considered a subclass of itself. *classinfo* may be a tuple of class objects, in which case every entry in *classinfo* will be checked. In any other @@ -830,6 +905,11 @@ are always available. They are listed here in alphabetical order. sequence (such as a string, bytes, tuple, list, or range) or a collection (such as a dictionary, set, or frozen set). + .. impl-detail:: + + ``len`` raises :exc:`OverflowError` on lengths larger than + :data:`sys.maxsize`, such as :class:`range(2 ** 100) `. + .. _func-list: .. class:: list([iterable]) @@ -890,7 +970,7 @@ are always available. They are listed here in alphabetical order. .. _func-memoryview: -.. function:: memoryview(obj) +.. class:: memoryview(obj) :noindex: Return a "memory view" object created from the given argument. See @@ -975,7 +1055,8 @@ are always available. They are listed here in alphabetical order. .. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None) Open *file* and return a corresponding :term:`file object`. If the file - cannot be opened, an :exc:`OSError` is raised. + cannot be opened, an :exc:`OSError` is raised. See + :ref:`tut-files` for more examples of how to use this function. *file* is a :term:`path-like object` giving the pathname (absolute or relative to the current working directory) of the file to be opened or an @@ -1008,12 +1089,12 @@ are always available. They are listed here in alphabetical order. ``'a'`` open for writing, appending to the end of the file if it exists ``'b'`` binary mode ``'t'`` text mode (default) - ``'+'`` open a disk file for updating (reading and writing) + ``'+'`` open for updating (reading and writing) ========= =============================================================== The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``). - For binary read-write access, the mode ``'w+b'`` opens and truncates the file - to 0 bytes. ``'r+b'`` opens the file without truncation. + Modes ``'w+'`` and ``'w+b'`` open and truncate the file. Modes ``'r+'`` + and ``'r+b'`` open the file with no truncation. As mentioned in the :ref:`io-overview`, Python distinguishes between binary and text I/O. Files opened in binary mode (including ``'b'`` in the *mode* @@ -1166,6 +1247,11 @@ are always available. They are listed here in alphabetical order. (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, and :mod:`shutil`. + .. audit-event:: open file,mode,flags open + + The ``mode`` and ``flags`` arguments may have been modified or inferred from + the original call. + .. versionchanged:: 3.3 @@ -1180,7 +1266,7 @@ are always available. They are listed here in alphabetical order. * The file is now non-inheritable. - .. deprecated-removed:: 3.4 4.0 + .. deprecated-removed:: 3.4 3.10 The ``'U'`` mode. @@ -1207,20 +1293,41 @@ are always available. They are listed here in alphabetical order. returns ``8364``. This is the inverse of :func:`chr`. -.. function:: pow(x, y[, z]) +.. function:: pow(base, exp[, mod]) - Return *x* to the power *y*; if *z* is present, return *x* to the power *y*, - modulo *z* (computed more efficiently than ``pow(x, y) % z``). The two-argument - form ``pow(x, y)`` is equivalent to using the power operator: ``x**y``. + Return *base* to the power *exp*; if *mod* is present, return *base* to the + power *exp*, modulo *mod* (computed more efficiently than + ``pow(base, exp) % mod``). The two-argument form ``pow(base, exp)`` is + equivalent to using the power operator: ``base**exp``. The arguments must have numeric types. With mixed operand types, the coercion rules for binary arithmetic operators apply. For :class:`int` operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are converted to float and a float result is delivered. For example, ``10**2`` - returns ``100``, but ``10**-2`` returns ``0.01``. If the second argument is - negative, the third argument must be omitted. If *z* is present, *x* and *y* - must be of integer types, and *y* must be non-negative. + returns ``100``, but ``10**-2`` returns ``0.01``. + + For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must + also be of integer type and *mod* must be nonzero. If *mod* is present and + *exp* is negative, *base* must be relatively prime to *mod*. In that case, + ``pow(inv_base, -exp, mod)`` is returned, where *inv_base* is an inverse to + *base* modulo *mod*. + + Here's an example of computing an inverse for ``38`` modulo ``97``:: + + >>> pow(38, -1, mod=97) + 23 + >>> 23 * 38 % 97 == 1 + True + + .. versionchanged:: 3.8 + For :class:`int` operands, the three-argument form of ``pow`` now allows + the second argument to be negative, permitting computation of modular + inverses. + + .. versionchanged:: 3.8 + Allow keyword arguments. Formerly, only positional arguments were + supported. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) @@ -1326,7 +1433,7 @@ are always available. They are listed here in alphabetical order. .. _func-range: -.. function:: range(stop) +.. class:: range(stop) range(start, stop[, step]) :noindex: @@ -1488,11 +1595,11 @@ are always available. They are listed here in alphabetical order. about strings, see :ref:`textseq`. -.. function:: sum(iterable[, start]) +.. function:: sum(iterable, /, start=0) Sums *start* and the items of an *iterable* from left to right and returns the - total. *start* defaults to ``0``. The *iterable*'s items are normally numbers, - and the start value is not allowed to be a string. + total. The *iterable*'s items are normally numbers, and the start value is not + allowed to be a string. For some use cases, there are good alternatives to :func:`sum`. The preferred, fast way to concatenate a sequence of strings is by calling @@ -1507,10 +1614,17 @@ are always available. They are listed here in alphabetical order. Return a proxy object that delegates method calls to a parent or sibling class of *type*. This is useful for accessing inherited methods that have - been overridden in a class. The search order is same as that used by - :func:`getattr` except that the *type* itself is skipped. + been overridden in a class. + + The *object-or-type* determines the :term:`method resolution order` + to be searched. The search starts from the class right after the + *type*. - The :attr:`~class.__mro__` attribute of the *type* lists the method + For example, if :attr:`~class.__mro__` of *object-or-type* is + ``D -> B -> C -> A -> object`` and the value of *type* is ``B``, + then :func:`super` searches ``C -> A -> object``. + + The :attr:`~class.__mro__` attribute of the *object-or-type* lists the method resolution search order used by both :func:`getattr` and :func:`super`. The attribute is dynamic and can change whenever the inheritance hierarchy is updated. @@ -1542,6 +1656,10 @@ are always available. They are listed here in alphabetical order. super().method(arg) # This does the same thing as: # super(C, self).method(arg) + In addition to method lookups, :func:`super` also works for attribute + lookups. One possible use case for this is calling :term:`descriptors ` + in a parent or sibling class. + Note that :func:`super` is implemented as part of the binding process for explicit dotted attribute lookups such as ``super().__getitem__(name)``. It does so by implementing its own :meth:`__getattribute__` method for searching @@ -1562,7 +1680,7 @@ are always available. They are listed here in alphabetical order. .. _func-tuple: -.. function:: tuple([iterable]) +.. class:: tuple([iterable]) :noindex: Rather than being a function, :class:`tuple` is actually an immutable @@ -1616,51 +1734,94 @@ are always available. They are listed here in alphabetical order. locals dictionary is only useful for reads since updates to the locals dictionary are ignored. + A :exc:`TypeError` exception is raised if an object is specified but + it doesn't have a :attr:`~object.__dict__` attribute (for example, if + its class defines the :attr:`~object.__slots__` attribute). -.. function:: zip(*iterables) - - Make an iterator that aggregates elements from each of the iterables. - - Returns an iterator of tuples, where the *i*-th tuple contains - the *i*-th element from each of the argument sequences or iterables. The - iterator stops when the shortest input iterable is exhausted. With a single - iterable argument, it returns an iterator of 1-tuples. With no arguments, - it returns an empty iterator. Equivalent to:: - - def zip(*iterables): - # zip('ABCD', 'xy') --> Ax By - sentinel = object() - iterators = [iter(it) for it in iterables] - while iterators: - result = [] - for it in iterators: - elem = next(it, sentinel) - if elem is sentinel: - return - result.append(elem) - yield tuple(result) - - The left-to-right evaluation order of the iterables is guaranteed. This - makes possible an idiom for clustering a data series into n-length groups - using ``zip(*[iter(s)]*n)``. This repeats the *same* iterator ``n`` times - so that each output tuple has the result of ``n`` calls to the iterator. - This has the effect of dividing the input into n-length chunks. - - :func:`zip` should only be used with unequal length inputs when you don't - care about trailing, unmatched values from the longer iterables. If those - values are important, use :func:`itertools.zip_longest` instead. - - :func:`zip` in conjunction with the ``*`` operator can be used to unzip a - list:: - - >>> x = [1, 2, 3] - >>> y = [4, 5, 6] - >>> zipped = zip(x, y) - >>> list(zipped) - [(1, 4), (2, 5), (3, 6)] - >>> x2, y2 = zip(*zip(x, y)) - >>> x == list(x2) and y == list(y2) - True +.. function:: zip(*iterables, strict=False) + + Iterate over several iterables in parallel, producing tuples with an item + from each one. + + Example:: + + >>> for item in zip([1, 2, 3], ['sugar', 'spice', 'everything nice']): + ... print(item) + ... + (1, 'sugar') + (2, 'spice') + (3, 'everything nice') + + More formally: :func:`zip` returns an iterator of tuples, where the *i*-th + tuple contains the *i*-th element from each of the argument iterables. + + Another way to think of :func:`zip` is that it turns rows into columns, and + columns into rows. This is similar to `transposing a matrix + `_. + + :func:`zip` is lazy: The elements won't be processed until the iterable is + iterated on, e.g. by a :keyword:`!for` loop or by wrapping in a + :class:`list`. + + One thing to consider is that the iterables passed to :func:`zip` could have + different lengths; sometimes by design, and sometimes because of a bug in + the code that prepared these iterables. Python offers three different + approaches to dealing with this issue: + + * By default, :func:`zip` stops when the shortest iterable is exhausted. + It will ignore the remaining items in the longer iterables, cutting off + the result to the length of the shortest iterable:: + + >>> list(zip(range(3), ['fee', 'fi', 'fo', 'fum'])) + [(0, 'fee'), (1, 'fi'), (2, 'fo')] + + * :func:`zip` is often used in cases where the iterables are assumed to be + of equal length. In such cases, it's recommended to use the ``strict=True`` + option. Its output is the same as regular :func:`zip`:: + + >>> list(zip(('a', 'b', 'c'), (1, 2, 3), strict=True)) + [('a', 1), ('b', 2), ('c', 3)] + + Unlike the default behavior, it checks that the lengths of iterables are + identical, raising a :exc:`ValueError` if they aren't: + + >>> list(zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True)) + Traceback (most recent call last): + ... + ValueError: zip() argument 2 is longer than argument 1 + + Without the ``strict=True`` argument, any bug that results in iterables of + different lengths will be silenced, possibly manifesting as a hard-to-find + bug in another part of the program. + + * Shorter iterables can be padded with a constant value to make all the + iterables have the same length. This is done by + :func:`itertools.zip_longest`. + + Edge cases: With a single iterable argument, :func:`zip` returns an + iterator of 1-tuples. With no arguments, it returns an empty iterator. + + Tips and tricks: + + * The left-to-right evaluation order of the iterables is guaranteed. This + makes possible an idiom for clustering a data series into n-length groups + using ``zip(*[iter(s)]*n, strict=True)``. This repeats the *same* iterator + ``n`` times so that each output tuple has the result of ``n`` calls to the + iterator. This has the effect of dividing the input into n-length chunks. + + * :func:`zip` in conjunction with the ``*`` operator can be used to unzip a + list:: + + >>> x = [1, 2, 3] + >>> y = [4, 5, 6] + >>> list(zip(x, y)) + [(1, 4), (2, 5), (3, 6)] + >>> x2, y2 = zip(*zip(x, y)) + >>> x == list(x2) and y == list(y2) + True + + .. versionchanged:: 3.10 + Added the ``strict`` argument. .. function:: __import__(name, globals=None, locals=None, fromlist=(), level=0) @@ -1731,6 +1892,9 @@ are always available. They are listed here in alphabetical order. Negative values for *level* are no longer supported (which also changes the default value to 0). + .. versionchanged:: 3.9 + When the command line options :option:`-E` or :option:`-I` are being used, + the environment variable :envvar:`PYTHONCASEOK` is now ignored. .. rubric:: Footnotes diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 16a779fa836858..14aa184e2cd14c 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -8,10 +8,16 @@ .. moduleauthor:: Raymond Hettinger .. moduleauthor:: Nick Coghlan .. moduleauthor:: Åukasz Langa +.. moduleauthor:: Pablo Galindo .. sectionauthor:: Peter Harris **Source code:** :source:`Lib/functools.py` +.. testsetup:: default + + import functools + from functools import * + -------------- The :mod:`functools` module is for higher-order functions: functions that act on @@ -20,6 +26,32 @@ function for the purposes of this module. The :mod:`functools` module defines the following functions: +.. decorator:: cache(user_function) + + Simple lightweight unbounded function cache. Sometimes called + `"memoize" `_. + + Returns the same as ``lru_cache(maxsize=None)``, creating a thin + wrapper around a dictionary lookup for the function arguments. Because it + never needs to evict old values, this is smaller and faster than + :func:`lru_cache()` with a size limit. + + For example:: + + @cache + def factorial(n): + return n * factorial(n-1) if n else 1 + + >>> factorial(10) # no previously cached result, makes 11 recursive calls + 3628800 + >>> factorial(5) # just looks up cached value result + 120 + >>> factorial(12) # makes two new recursive calls, the other 10 are cached + 479001600 + + .. versionadded:: 3.9 + + .. decorator:: cached_property(func) Transform a method of a class into a property whose value is computed once @@ -76,7 +108,8 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.2 -.. decorator:: lru_cache(maxsize=128, typed=False) +.. decorator:: lru_cache(user_function) + lru_cache(maxsize=128, typed=False) Decorator to wrap a function with a memoizing callable that saves up to the *maxsize* most recent calls. It can save time when an expensive or I/O bound @@ -90,14 +123,27 @@ The :mod:`functools` module defines the following functions: differ in their keyword argument order and may have two separate cache entries. + If *user_function* is specified, it must be a callable. This allows the + *lru_cache* decorator to be applied directly to a user function, leaving + the *maxsize* at its default value of 128:: + + @lru_cache + def count_vowels(sentence): + sentence = sentence.casefold() + return sum(sentence.count(vowel) for vowel in 'aeiou') + If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can - grow without bound. The LRU feature performs best when *maxsize* is a - power-of-two. + grow without bound. If *typed* is set to true, function arguments of different types will be cached separately. For example, ``f(3)`` and ``f(3.0)`` will be treated as distinct calls with distinct results. + The wrapped function is instrumented with a :func:`cache_parameters` + function that returns a new :class:`dict` showing the values for *maxsize* + and *typed*. This is for information purposes only. Mutating the values + has no effect. + To help measure the effectiveness of the cache and tune the *maxsize* parameter, the wrapped function is instrumented with a :func:`cache_info` function that returns a :term:`named tuple` showing *hits*, *misses*, @@ -112,11 +158,11 @@ The :mod:`functools` module defines the following functions: bypassing the cache, or for rewrapping the function with a different cache. An `LRU (least recently used) cache - `_ works - best when the most recent calls are the best predictors of upcoming calls (for - example, the most popular articles on a news server tend to change each day). - The cache's size limit assures that the cache does not grow without bound on - long-running processes such as web servers. + `_ + works best when the most recent calls are the best predictors of upcoming + calls (for example, the most popular articles on a news server tend to + change each day). The cache's size limit assures that the cache does not + grow without bound on long-running processes such as web servers. In general, the LRU cache should only be used when you want to reuse previously computed values. Accordingly, it doesn't make sense to cache @@ -165,6 +211,12 @@ The :mod:`functools` module defines the following functions: .. versionchanged:: 3.3 Added the *typed* option. + .. versionchanged:: 3.8 + Added the *user_function* option. + + .. versionadded:: 3.9 + Added the function :func:`cache_parameters` + .. decorator:: total_ordering Given a class defining one or more rich comparison ordering methods, this @@ -208,7 +260,7 @@ The :mod:`functools` module defines the following functions: Returning NotImplemented from the underlying comparison function for unrecognised types is now supported. -.. function:: partial(func, *args, **keywords) +.. function:: partial(func, /, *args, **keywords) Return a new :ref:`partial object` which when called will behave like *func* called with the positional arguments *args* @@ -217,7 +269,7 @@ The :mod:`functools` module defines the following functions: supplied, they extend and override *keywords*. Roughly equivalent to:: - def partial(func, *args, **keywords): + def partial(func, /, *args, **keywords): def newfunc(*fargs, **fkeywords): newkeywords = {**keywords, **fkeywords} return func(*args, *fargs, **newkeywords) @@ -239,7 +291,7 @@ The :mod:`functools` module defines the following functions: 18 -.. class:: partialmethod(func, *args, **keywords) +.. class:: partialmethod(func, /, *args, **keywords) Return a new :class:`partialmethod` descriptor which behaves like :class:`partial` except that it is designed to be used as a method @@ -262,7 +314,7 @@ The :mod:`functools` module defines the following functions: Example:: - >>> class Cell(object): + >>> class Cell: ... def __init__(self): ... self._alive = False ... @property @@ -402,6 +454,20 @@ The :mod:`functools` module defines the following functions: for the base ``object`` type, which means it is used if no better implementation is found. + If an implementation registered to :term:`abstract base class`, virtual + subclasses will be dispatched to that implementation:: + + >>> from collections.abc import Mapping + >>> @fun.register + ... def _(arg: Mapping, verbose=False): + ... if verbose: + ... print("Keys & Values") + ... for key, value in arg.items(): + ... print(key, "=>", value) + ... + >>> fun({"a": "b"}) + a => b + To check which implementation will the generic function choose for a given type, use the ``dispatch()`` attribute:: @@ -585,4 +651,4 @@ callable, weak referencable, and can have attributes. There are some important differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes are not created automatically. Also, :class:`partial` objects defined in classes behave like static methods and do not transform into bound methods -during instance attribute look-up. +during instance attribute look-up. \ No newline at end of file diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 084cd6ac257ebe..2d85cd3431711a 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -35,7 +35,7 @@ The :mod:`gc` module provides the following functions: .. function:: isenabled() - Returns true if automatic collection is enabled. + Return ``True`` if automatic collection is enabled. .. function:: collect(generation=2) @@ -106,9 +106,9 @@ The :mod:`gc` module provides the following functions: allocations minus the number of deallocations exceeds *threshold0*, collection starts. Initially only generation ``0`` is examined. If generation ``0`` has been examined more than *threshold1* times since generation ``1`` has been - examined, then generation ``1`` is examined as well. Similarly, *threshold2* - controls the number of collections of generation ``1`` before collecting - generation ``2``. + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. .. function:: get_count() @@ -177,6 +177,27 @@ The :mod:`gc` module provides the following functions: .. versionadded:: 3.1 +.. function:: is_finalized(obj) + + Returns ``True`` if the given object has been finalized by the + garbage collector, ``False`` otherwise. :: + + >>> x = None + >>> class Lazarus: + ... def __del__(self): + ... global x + ... x = self + ... + >>> lazarus = Lazarus() + >>> gc.is_finalized(lazarus) + False + >>> del lazarus + >>> gc.is_finalized(x) + True + + .. versionadded:: 3.9 + + .. function:: freeze() Freeze all the objects tracked by gc - move them to a permanent generation @@ -212,7 +233,7 @@ values but should not rebind them): A list of objects which the collector found to be unreachable but could not be freed (uncollectable objects). Starting with Python 3.4, this list should be empty most of the time, except when using instances of - C extension types with a non-NULL ``tp_del`` slot. + C extension types with a non-``NULL`` ``tp_del`` slot. If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be added to this list rather than freed. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 7f4eab5843f5f2..ec2c12806b4160 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -39,7 +39,7 @@ class-based API instead. Bind the *domain* to the locale directory *localedir*. More concretely, :mod:`gettext` will look for binary :file:`.mo` files for the given domain using the path (on Unix): :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo`, where - *languages* is searched for in the environment variables :envvar:`LANGUAGE`, + *language* is searched for in the environment variables :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. If *localedir* is omitted or ``None``, then the current binding for *domain* is @@ -724,8 +724,8 @@ implementations, and valuable experience to the creation of this module: .. [#] The default locale directory is system dependent; for example, on RedHat Linux it is :file:`/usr/share/locale`, but on Solaris it is :file:`/usr/lib/locale`. The :mod:`gettext` module does not try to support these system dependent - defaults; instead its default is :file:`{sys.prefix}/share/locale` (see - :data:`sys.prefix`). For this reason, it is always best to call + defaults; instead its default is :file:`{sys.base_prefix}/share/locale` (see + :data:`sys.base_prefix`). For this reason, it is always best to call :func:`bindtextdomain` with an explicit absolute path at the start of your application. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 2a5f0ddc4ef319..3fdba6937c1de0 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -36,21 +36,35 @@ For example, ``'[?]'`` matches the character ``'?'``. The :mod:`pathlib` module offers high-level path objects. -.. function:: glob(pathname, *, recursive=False) +.. function:: glob(pathname, *, root_dir=None, dir_fd=None, recursive=False) Return a possibly-empty list of path names that match *pathname*, which must be a string containing a path specification. *pathname* can be either absolute (like :file:`/usr/src/Python-1.5/Makefile`) or relative (like :file:`../../Tools/\*/\*.gif`), and can contain shell-style wildcards. Broken symlinks are included in the results (as in the shell). Whether or not the - results are sorted depends on the file system. + results are sorted depends on the file system. If a file that satisfies + conditions is removed or added during the call of this function, whether + a path name for that file be included is unspecified. + + If *root_dir* is not ``None``, it should be a :term:`path-like object` + specifying the root directory for searching. It has the same effect on + :func:`glob` as changing the current directory before calling it. If + *pathname* is relative, the result will contain paths relative to + *root_dir*. + + This function can support :ref:`paths relative to directory descriptors + ` with the *dir_fd* parameter. .. index:: single: **; in glob-style wildcards If *recursive* is true, the pattern "``**``" will match any files and zero or - more directories and subdirectories. If the pattern is followed by an - ``os.sep``, only directories and subdirectories match. + more directories, subdirectories and symbolic links to directories. If the + pattern is followed by an :data:`os.sep` or :data:`os.altsep` then files will not + match. + + .. audit-event:: glob.glob pathname,recursive glob.glob .. note:: Using the "``**``" pattern in large directory trees may consume @@ -59,12 +73,17 @@ For example, ``'[?]'`` matches the character ``'?'``. .. versionchanged:: 3.5 Support for recursive globs using "``**``". + .. versionchanged:: 3.10 + Added the *root_dir* and *dir_fd* parameters. -.. function:: iglob(pathname, *, recursive=False) + +.. function:: iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False) Return an :term:`iterator` which yields the same values as :func:`glob` without actually storing them all simultaneously. + .. audit-event:: glob.glob pathname,recursive glob.iglob + .. function:: escape(pathname) diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst new file mode 100644 index 00000000000000..0faca2186b268c --- /dev/null +++ b/Doc/library/graphlib.rst @@ -0,0 +1,209 @@ +:mod:`graphlib` --- Functionality to operate with graph-like structures +========================================================================= + +.. module:: graphlib + :synopsis: Functionality to operate with graph-like structures + + +**Source code:** :source:`Lib/graphlib.py` + +.. testsetup:: default + + import graphlib + from graphlib import * + +-------------- + + +.. class:: TopologicalSorter(graph=None) + + Provides functionality to topologically sort a graph of hashable nodes. + + A topological order is a linear ordering of the vertices in a graph such that + for every directed edge u -> v from vertex u to vertex v, vertex u comes + before vertex v in the ordering. For instance, the vertices of the graph may + represent tasks to be performed, and the edges may represent constraints that + one task must be performed before another; in this example, a topological + ordering is just a valid sequence for the tasks. A complete topological + ordering is possible if and only if the graph has no directed cycles, that + is, if it is a directed acyclic graph. + + If the optional *graph* argument is provided it must be a dictionary + representing a directed acyclic graph where the keys are nodes and the values + are iterables of all predecessors of that node in the graph (the nodes that + have edges that point to the value in the key). Additional nodes can be added + to the graph using the :meth:`~TopologicalSorter.add` method. + + In the general case, the steps required to perform the sorting of a given + graph are as follows: + + * Create an instance of the :class:`TopologicalSorter` with an optional + initial graph. + * Add additional nodes to the graph. + * Call :meth:`~TopologicalSorter.prepare` on the graph. + * While :meth:`~TopologicalSorter.is_active` is ``True``, iterate over + the nodes returned by :meth:`~TopologicalSorter.get_ready` and + process them. Call :meth:`~TopologicalSorter.done` on each node as it + finishes processing. + + In case just an immediate sorting of the nodes in the graph is required and + no parallelism is involved, the convenience method + :meth:`TopologicalSorter.static_order` can be used directly: + + .. doctest:: + + >>> graph = {"D": {"B", "C"}, "C": {"A"}, "B": {"A"}} + >>> ts = TopologicalSorter(graph) + >>> tuple(ts.static_order()) + ('A', 'C', 'B', 'D') + + The class is designed to easily support parallel processing of the nodes as + they become ready. For instance:: + + topological_sorter = TopologicalSorter() + + # Add nodes to 'topological_sorter'... + + topological_sorter.prepare() + while topological_sorter.is_active(): + for node in topological_sorter.get_ready(): + # Worker threads or processes take nodes to work on off the + # 'task_queue' queue. + task_queue.put(node) + + # When the work for a node is done, workers put the node in + # 'finalized_tasks_queue' so we can get more nodes to work on. + # The definition of 'is_active()' guarantees that, at this point, at + # least one node has been placed on 'task_queue' that hasn't yet + # been passed to 'done()', so this blocking 'get()' must (eventually) + # succeed. After calling 'done()', we loop back to call 'get_ready()' + # again, so put newly freed nodes on 'task_queue' as soon as + # logically possible. + node = finalized_tasks_queue.get() + topological_sorter.done(node) + + .. method:: add(node, *predecessors) + + Add a new node and its predecessors to the graph. Both the *node* and all + elements in *predecessors* must be hashable. + + If called multiple times with the same node argument, the set of + dependencies will be the union of all dependencies passed in. + + It is possible to add a node with no dependencies (*predecessors* is not + provided) or to provide a dependency twice. If a node that has not been + provided before is included among *predecessors* it will be automatically + added to the graph with no predecessors of its own. + + Raises :exc:`ValueError` if called after :meth:`~TopologicalSorter.prepare`. + + .. method:: prepare() + + Mark the graph as finished and check for cycles in the graph. If any cycle + is detected, :exc:`CycleError` will be raised, but + :meth:`~TopologicalSorter.get_ready` can still be used to obtain as many + nodes as possible until cycles block more progress. After a call to this + function, the graph cannot be modified, and therefore no more nodes can be + added using :meth:`~TopologicalSorter.add`. + + .. method:: is_active() + + Returns ``True`` if more progress can be made and ``False`` otherwise. + Progress can be made if cycles do not block the resolution and either + there are still nodes ready that haven't yet been returned by + :meth:`TopologicalSorter.get_ready` or the number of nodes marked + :meth:`TopologicalSorter.done` is less than the number that have been + returned by :meth:`TopologicalSorter.get_ready`. + + The :meth:`~TopologicalSorter.__bool__` method of this class defers to + this function, so instead of:: + + if ts.is_active(): + ... + + it is possible to simply do:: + + if ts: + ... + + Raises :exc:`ValueError` if called without calling + :meth:`~TopologicalSorter.prepare` previously. + + .. method:: done(*nodes) + + Marks a set of nodes returned by :meth:`TopologicalSorter.get_ready` as + processed, unblocking any successor of each node in *nodes* for being + returned in the future by a call to :meth:`TopologicalSorter.get_ready`. + + Raises :exc:`ValueError` if any node in *nodes* has already been marked as + processed by a previous call to this method or if a node was not added to + the graph by using :meth:`TopologicalSorter.add`, if called without + calling :meth:`~TopologicalSorter.prepare` or if node has not yet been + returned by :meth:`~TopologicalSorter.get_ready`. + + .. method:: get_ready() + + Returns a ``tuple`` with all the nodes that are ready. Initially it + returns all nodes with no predecessors, and once those are marked as + processed by calling :meth:`TopologicalSorter.done`, further calls will + return all new nodes that have all their predecessors already processed. + Once no more progress can be made, empty tuples are returned. + + Raises :exc:`ValueError` if called without calling + :meth:`~TopologicalSorter.prepare` previously. + + .. method:: static_order() + + Returns an iterable of nodes in a topological order. Using this method + does not require to call :meth:`TopologicalSorter.prepare` or + :meth:`TopologicalSorter.done`. This method is equivalent to:: + + def static_order(self): + self.prepare() + while self.is_active(): + node_group = self.get_ready() + yield from node_group + self.done(*node_group) + + The particular order that is returned may depend on the specific order in + which the items were inserted in the graph. For example: + + .. doctest:: + + >>> ts = TopologicalSorter() + >>> ts.add(3, 2, 1) + >>> ts.add(1, 0) + >>> print([*ts.static_order()]) + [2, 0, 1, 3] + + >>> ts2 = TopologicalSorter() + >>> ts2.add(1, 0) + >>> ts2.add(3, 2, 1) + >>> print([*ts2.static_order()]) + [0, 2, 1, 3] + + This is due to the fact that "0" and "2" are in the same level in the + graph (they would have been returned in the same call to + :meth:`~TopologicalSorter.get_ready`) and the order between them is + determined by the order of insertion. + + + If any cycle is detected, :exc:`CycleError` will be raised. + + .. versionadded:: 3.9 + + +Exceptions +---------- +The :mod:`graphlib` module defines the following exception classes: + +.. exception:: CycleError + + Subclass of :exc:`ValueError` raised by :meth:`TopologicalSorter.prepare` if cycles exist + in the working graph. If multiple cycles exist, only one undefined choice among them will + be reported and included in the exception. + + The detected cycle can be accessed via the second element in the :attr:`~CycleError.args` + attribute of the exception instance and consists in a list of nodes, such that each node is, + in the graph, an immediate predecessor of the next node in the list. In the reported list, + the first and the last node will be the same, to make it clear that it is cyclic. \ No newline at end of file diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 8850a33f4abb90..33c40676f747c5 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -59,6 +59,14 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. +.. exception:: BadGzipFile + + An exception raised for invalid gzip files. It inherits :exc:`OSError`. + :exc:`EOFError` and :exc:`zlib.error` can also be raised for invalid gzip + files. + + .. versionadded:: 3.8 + .. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None) Constructor for the :class:`GzipFile` class, which simulates most of the @@ -80,7 +88,8 @@ The module defines the following items: The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``, ``'w'``, ``'wb'``, ``'x'``, or ``'xb'``, depending on whether the file will be read or written. The default is the mode of *fileobj* if discernible; otherwise, the - default is ``'rb'``. + default is ``'rb'``. In future Python releases the mode of *fileobj* will + not be used. It is better to always specify *mode* for writing. Note that the file is always opened in binary mode. To open a compressed file in text mode, use :func:`.open` (or wrap your :class:`GzipFile` with an @@ -156,6 +165,10 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. deprecated:: 3.9 + Opening :class:`GzipFile` for writing without specifying the *mode* + argument is deprecated. + .. function:: compress(data, compresslevel=9, *, mtime=None) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index a16c7cd4d7cd6e..d644974e660984 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -67,7 +67,7 @@ Constructors for hash algorithms that are always present in this module are :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, :func:`sha512`, :func:`blake2b`, and :func:`blake2s`. :func:`md5` is normally available as well, though it -may be missing if you are using a rare "FIPS compliant" build of Python. +may be missing or blocked if you are using a rare "FIPS compliant" build of Python. Additional algorithms may also be available depending upon the OpenSSL library that Python uses on your platform. On most platforms the :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`, @@ -80,6 +80,15 @@ library that Python uses on your platform. On most platforms the .. versionadded:: 3.6 :func:`blake2b` and :func:`blake2s` were added. +.. versionchanged:: 3.9 + All hashlib constructors take a keyword-only argument *usedforsecurity* + with default value ``True``. A false value allows the use of insecure and + blocked hashing algorithms in restricted environments. ``False`` indicates + that the hashing algorithm is not used in a security context, e.g. as a + non-cryptographic one-way compression function. + + Hashlib now uses SHA3 and SHAKE from OpenSSL 1.1.1 and newer. + For example, to obtain the digest of the byte string ``b'Nobody inspects the spammish repetition'``:: @@ -99,7 +108,7 @@ More condensed: >>> hashlib.sha224(b"Nobody inspects the spammish repetition").hexdigest() 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' -.. function:: new(name[, data]) +.. function:: new(name[, data], *, usedforsecurity=True) Is a generic constructor that takes the string *name* of the desired algorithm as its first parameter. It also exists to allow access to the @@ -308,11 +317,13 @@ New hash objects are created by calling constructor functions: .. function:: blake2b(data=b'', *, digest_size=64, key=b'', salt=b'', \ person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ - node_depth=0, inner_size=0, last_node=False) + node_depth=0, inner_size=0, last_node=False, \ + usedforsecurity=True) .. function:: blake2s(data=b'', *, digest_size=32, key=b'', salt=b'', \ person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ - node_depth=0, inner_size=0, last_node=False) + node_depth=0, inner_size=0, last_node=False, \ + usedforsecurity=True) These functions return the corresponding hash objects for calculating diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index dc994b07c35c00..6f1b59b57ce580 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -14,12 +14,13 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. -.. function:: new(key, msg=None, digestmod=None) +.. function:: new(key, msg=None, digestmod='') Return a new hmac object. *key* is a bytes or bytearray object giving the secret key. If *msg* is present, the method call ``update(msg)`` is made. *digestmod* is the digest name, digest constructor or module for the HMAC - object to use. It supports any name suitable to :func:`hashlib.new`. + object to use. It may be any name suitable to :func:`hashlib.new`. + Despite its argument position, it is required. .. versionchanged:: 3.4 Parameter *key* can be a bytes or bytearray object. @@ -28,6 +29,8 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. .. deprecated-removed:: 3.4 3.8 MD5 as implicit default digest for *digestmod* is deprecated. + The digestmod parameter is now required. Pass it as a keyword + argument to avoid awkwardness when you do not have an initial msg. .. function:: digest(key, msg, digest) @@ -111,6 +114,12 @@ A hash object has the following attributes: .. versionadded:: 3.4 +.. deprecated:: 3.9 + + The undocumented attributes ``HMAC.digest_cons``, ``HMAC.inner``, and + ``HMAC.outer`` are internal implementation details and will be removed in + Python 3.10. + This module also provides the following helper function: .. function:: compare_digest(a, b) @@ -127,9 +136,13 @@ This module also provides the following helper function: a timing attack could theoretically reveal information about the types and lengths of *a* and *b*—but not their values. - .. versionadded:: 3.3 + .. versionchanged:: 3.10 + + The function uses OpenSSL's ``CRYPTO_memcmp()`` internally when + available. + .. seealso:: diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index beaa720d732b4c..35997db2a9d27d 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -20,7 +20,7 @@ HTTPS protocols. It is normally not used directly --- the module .. seealso:: - The `Requests package `_ + The `Requests package `_ is recommended for a higher-level HTTP client interface. .. note:: @@ -94,6 +94,11 @@ The module provides the following classes: :func:`ssl._create_unverified_context` can be passed to the *context* parameter. + .. versionchanged:: 3.8 + This class now enables TLS 1.3 + :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or + when *cert_file* is passed with a custom *context*. + .. deprecated:: 3.6 *key_file* and *cert_file* are deprecated in favor of *context*. @@ -479,6 +484,14 @@ statement. HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. +.. attribute:: HTTPResponse.url + + URL of the resource retrieved, commonly used to determine if a redirect was followed. + +.. attribute:: HTTPResponse.headers + + Headers of the response in the form of an :class:`email.message.EmailMessage` instance. + .. attribute:: HTTPResponse.status Status code returned by server. @@ -496,6 +509,21 @@ statement. Is ``True`` if the stream is closed. +.. method:: HTTPResponse.geturl() + + .. deprecated:: 3.9 + Deprecated in favor of :attr:`~HTTPResponse.url`. + +.. method:: HTTPResponse.info() + + .. deprecated:: 3.9 + Deprecated in favor of :attr:`~HTTPResponse.headers`. + +.. method:: HTTPResponse.getstatus() + + .. deprecated:: 3.9 + Deprecated in favor of :attr:`~HTTPResponse.status`. + Examples -------- @@ -511,8 +539,8 @@ Here is an example session that uses the ``GET`` method:: >>> # The following example demonstrates reading data in chunks. >>> conn.request("GET", "/") >>> r1 = conn.getresponse() - >>> while not r1.closed: - ... print(r1.read(200)) # 200 bytes + >>> while chunk := r1.read(200): + ... print(repr(chunk)) b'\n A B C D A B # unique_justseen('ABBCcAD', str.lower) --> A B C A D - return map(next, map(itemgetter(1), groupby(iterable, key))) + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) def iter_except(func, exception, first=None): """ Call a function repeatedly until an exception is raised. @@ -913,9 +926,3 @@ which incur interpreter overhead. result.append(pool[-1-n]) return tuple(result) -Note, many of the above recipes can be optimized by replacing global lookups -with local variables defined as default values. For example, the -*dotproduct* recipe can be written as:: - - def dotproduct(vec1, vec2, sum=sum, map=map, mul=operator.mul): - return sum(map(mul, vec1, vec2)) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index b476c372370d06..e1a246aad6190b 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -120,6 +120,18 @@ See :ref:`json-commandline` for detailed documentation. value) is also a subset of YAML 1.0 and 1.1. This module can thus also be used as a YAML serializer. +.. note:: + + This module's encoders and decoders preserve input and output order by + default. Order is only lost if the underlying containers are unordered. + + Prior to Python 3.7, :class:`dict` was not guaranteed to be ordered, so + inputs and outputs were typically scrambled unless + :class:`collections.OrderedDict` was specifically requested. Starting + with Python 3.7, the regular :class:`dict` became order preserving, so + it is no longer necessary to specify :class:`collections.OrderedDict` for + JSON generation and parsing. + Basic Usage ----------- @@ -271,19 +283,18 @@ Basic Usage instance containing a JSON document) to a Python object using this :ref:`conversion table `. - The other arguments have the same meaning as in :func:`load`, except - *encoding* which is ignored and deprecated since Python 3.1. + The other arguments have the same meaning as in :func:`load`. If the data being deserialized is not a valid JSON document, a :exc:`JSONDecodeError` will be raised. - .. deprecated-removed:: 3.1 3.9 - *encoding* keyword argument. - .. versionchanged:: 3.6 *s* can now be of type :class:`bytes` or :class:`bytearray`. The input encoding should be UTF-8, UTF-16 or UTF-32. + .. versionchanged:: 3.9 + The keyword argument *encoding* has been removed. + Encoders and Decoders --------------------- @@ -720,12 +731,24 @@ Command line options .. versionadded:: 3.5 +.. cmdoption:: --no-ensure-ascii + + Disable escaping of non-ascii characters, see :func:`json.dumps` for more information. + + .. versionadded:: 3.9 + .. cmdoption:: --json-lines Parse every input line as separate JSON object. .. versionadded:: 3.8 +.. cmdoption:: --indent, --tab, --no-indent, --compact + + Mutually exclusive options for whitespace control. + + .. versionadded:: 3.9 + .. cmdoption:: -h, --help Show the help message. diff --git a/Doc/library/keyword.rst b/Doc/library/keyword.rst index 173db23544a473..5cae79f5dc9dbc 100644 --- a/Doc/library/keyword.rst +++ b/Doc/library/keyword.rst @@ -8,16 +8,33 @@ -------------- -This module allows a Python program to determine if a string is a keyword. +This module allows a Python program to determine if a string is a +:ref:`keyword `. .. function:: iskeyword(s) - Return true if *s* is a Python keyword. + Return ``True`` if *s* is a Python :ref:`keyword `. .. data:: kwlist - Sequence containing all the keywords defined for the interpreter. If any - keywords are defined to only be active when particular :mod:`__future__` - statements are in effect, these will be included as well. + Sequence containing all the :ref:`keywords ` defined for the + interpreter. If any keywords are defined to only be active when particular + :mod:`__future__` statements are in effect, these will be included as well. + + +.. function:: issoftkeyword(s) + + Return ``True`` if *s* is a Python soft :ref:`keyword `. + + .. versionadded:: 3.9 + + +.. data:: softkwlist + + Sequence containing all the soft :ref:`keywords ` defined for the + interpreter. If any soft keywords are defined to only be active when particular + :mod:`__future__` statements are in effect, these will be included as well. + + .. versionadded:: 3.9 diff --git a/Doc/library/language.rst b/Doc/library/language.rst index 1eac32e4590216..510f064c76d795 100644 --- a/Doc/library/language.rst +++ b/Doc/library/language.rst @@ -13,10 +13,8 @@ These modules include: .. toctree:: - parser.rst ast.rst symtable.rst - symbol.rst token.rst keyword.rst tokenize.rst diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index 9278fc0641fca8..dd9f4ee45ba82e 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -2,7 +2,7 @@ ================================================ .. module:: linecache - :synopsis: This module provides random access to individual lines from text files. + :synopsis: Provides random access to individual lines from text files. .. sectionauthor:: Moshe Zadka diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index bf57a083559168..678148a0dda294 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -508,7 +508,7 @@ Background, details, hints, tips and caveats -------------------------------------------- The C standard defines the locale as a program-wide property that may be -relatively expensive to change. On top of that, some implementation are broken +relatively expensive to change. On top of that, some implementations are broken in such a way that frequent locale changes may cause core dumps. This makes the locale somewhat painful to use correctly. diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index dee9a84e333764..059ab3d3a34448 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -48,7 +48,7 @@ and :meth:`flush` methods). .. method:: emit(record) If a formatter is specified, it is used to format the record. The record - is then written to the stream with a terminator. If exception information + is then written to the stream followed by :attr:`terminator`. If exception information is present, it is formatted using :func:`traceback.print_exception` and appended to the stream. @@ -68,15 +68,19 @@ and :meth:`flush` methods). :return: the old stream, if the stream was changed, or *None* if it wasn't. - .. versionadded:: 3.7 + .. versionadded:: 3.7 + .. attribute:: terminator -.. versionchanged:: 3.2 - The ``StreamHandler`` class now has a ``terminator`` attribute, default - value ``'\n'``, which is used as the terminator when writing a formatted - record to a stream. If you don't want this newline termination, you can - set the handler instance's ``terminator`` attribute to the empty string. - In earlier versions, the terminator was hardcoded as ``'\n'``. + String used as the terminator when writing a formatted record to a stream. + Default value is ``'\n'``. + + If you don't want a newline termination, you can set the handler instance's + ``terminator`` attribute to the empty string. + + In earlier versions, the terminator was hardcoded as ``'\n'``. + + .. versionadded:: 3.2 .. _file-handler: @@ -89,23 +93,26 @@ sends logging output to a disk file. It inherits the output functionality from :class:`StreamHandler`. -.. class:: FileHandler(filename, mode='a', encoding=None, delay=False) +.. class:: FileHandler(filename, mode='a', encoding=None, delay=False, errors=None) Returns a new instance of the :class:`FileHandler` class. The specified file is opened and used as the stream for logging. If *mode* is not specified, :const:`'a'` is used. If *encoding* is not ``None``, it is used to open the file with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. + first call to :meth:`emit`. By default, the file grows indefinitely. If + *errors* is specified, it's used to determine how encoding errors are handled. .. versionchanged:: 3.6 As well as string values, :class:`~pathlib.Path` objects are also accepted for the *filename* argument. + .. versionchanged:: 3.9 + The *errors* parameter was added. + .. method:: close() Closes the file. - .. method:: emit(record) Outputs the record to the file. @@ -168,18 +175,22 @@ exclusive locks - and so there is no need for such a handler. Furthermore, for this value. -.. class:: WatchedFileHandler(filename, mode='a', encoding=None, delay=False) +.. class:: WatchedFileHandler(filename, mode='a', encoding=None, delay=False, errors=None) Returns a new instance of the :class:`WatchedFileHandler` class. The specified file is opened and used as the stream for logging. If *mode* is not specified, :const:`'a'` is used. If *encoding* is not ``None``, it is used to open the file with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. + first call to :meth:`emit`. By default, the file grows indefinitely. If + *errors* is provided, it determines how encoding errors are handled. .. versionchanged:: 3.6 As well as string values, :class:`~pathlib.Path` objects are also accepted for the *filename* argument. + .. versionchanged:: 3.9 + The *errors* parameter was added. + .. method:: reopenIfNeeded() Checks to see if the file has changed. If it has, the existing stream is @@ -205,7 +216,7 @@ module, is the base class for the rotating file handlers, not need to instantiate this class, but it has attributes and methods you may need to override. -.. class:: BaseRotatingHandler(filename, mode, encoding=None, delay=False) +.. class:: BaseRotatingHandler(filename, mode, encoding=None, delay=False, errors=None) The parameters are as for :class:`FileHandler`. The attributes are: @@ -284,13 +295,14 @@ The :class:`RotatingFileHandler` class, located in the :mod:`logging.handlers` module, supports rotation of disk log files. -.. class:: RotatingFileHandler(filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False) +.. class:: RotatingFileHandler(filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False, errors=None) Returns a new instance of the :class:`RotatingFileHandler` class. The specified file is opened and used as the stream for logging. If *mode* is not specified, ``'a'`` is used. If *encoding* is not ``None``, it is used to open the file with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. + first call to :meth:`emit`. By default, the file grows indefinitely. If + *errors* is provided, it determines how encoding errors are handled. You can use the *maxBytes* and *backupCount* values to allow the file to :dfn:`rollover` at a predetermined size. When the size is about to be exceeded, @@ -311,6 +323,9 @@ module, supports rotation of disk log files. As well as string values, :class:`~pathlib.Path` objects are also accepted for the *filename* argument. + .. versionchanged:: 3.9 + The *errors* parameter was added. + .. method:: doRollover() Does a rollover, as described above. @@ -331,7 +346,7 @@ The :class:`TimedRotatingFileHandler` class, located in the timed intervals. -.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None) +.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None, errors=None) Returns a new instance of the :class:`TimedRotatingFileHandler` class. The specified file is opened and used as the stream for logging. On rotating it also @@ -391,6 +406,9 @@ timed intervals. rollover, and subsequent rollovers would be calculated via the normal interval calculation. + If *errors* is specified, it's used to determine how encoding errors are + handled. + .. note:: Calculation of the initial rollover time is done when the handler is initialised. Calculation of subsequent rollover times is done only when rollover occurs, and rollover occurs only when emitting output. If @@ -411,6 +429,9 @@ timed intervals. As well as string values, :class:`~pathlib.Path` objects are also accepted for the *filename* argument. + .. versionchanged:: 3.9 + The *errors* parameter was added. + .. method:: doRollover() Does a rollover, as described above. @@ -470,7 +491,12 @@ sends logging output to a network socket. The base class uses a TCP socket. .. method:: makePickle(record) Pickles the record's attribute dictionary in binary format with a length - prefix, and returns it ready for transmission across the socket. + prefix, and returns it ready for transmission across the socket. The + details of this operation are equivalent to:: + + data = pickle.dumps(record_attr_dict, 1) + datalen = struct.pack('>L', len(data)) + return datalen + data Note that pickles aren't completely secure. If you are concerned about security, you may want to override this method to implement a more secure @@ -481,8 +507,12 @@ sends logging output to a network socket. The base class uses a TCP socket. .. method:: send(packet) - Send a pickled string *packet* to the socket. This function allows for - partial sends which can happen when the network is busy. + Send a pickled byte-string *packet* to the socket. The format of the sent + byte-string is as described in the documentation for + :meth:`~SocketHandler.makePickle`. + + This function allows for partial sends, which can happen when the network + is busy. .. method:: createSocket() @@ -543,7 +573,8 @@ over UDP sockets. .. method:: send(s) - Send a pickled string to a socket. + Send a pickled byte-string to a socket. The format of the sent byte-string + is as described in the documentation for :meth:`SocketHandler.makePickle`. .. _syslog-handler: @@ -830,13 +861,14 @@ should, then :meth:`flush` is expected to do the flushing. .. class:: BufferingHandler(capacity) - Initializes the handler with a buffer of the specified capacity. + Initializes the handler with a buffer of the specified capacity. Here, + *capacity* means the number of logging records buffered. .. method:: emit(record) - Appends the record to the buffer. If :meth:`shouldFlush` returns true, - calls :meth:`flush` to process the buffer. + Append the record to the buffer. If :meth:`shouldFlush` returns true, + call :meth:`flush` to process the buffer. .. method:: flush() @@ -847,19 +879,20 @@ should, then :meth:`flush` is expected to do the flushing. .. method:: shouldFlush(record) - Returns true if the buffer is up to capacity. This method can be + Return ``True`` if the buffer is up to capacity. This method can be overridden to implement custom flushing strategies. .. class:: MemoryHandler(capacity, flushLevel=ERROR, target=None, flushOnClose=True) Returns a new instance of the :class:`MemoryHandler` class. The instance is - initialized with a buffer size of *capacity*. If *flushLevel* is not specified, - :const:`ERROR` is used. If no *target* is specified, the target will need to be - set using :meth:`setTarget` before this handler does anything useful. If - *flushOnClose* is specified as ``False``, then the buffer is *not* flushed when - the handler is closed. If not specified or specified as ``True``, the previous - behaviour of flushing the buffer will occur when the handler is closed. + initialized with a buffer size of *capacity* (number of records buffered). + If *flushLevel* is not specified, :const:`ERROR` is used. If no *target* is + specified, the target will need to be set using :meth:`setTarget` before this + handler does anything useful. If *flushOnClose* is specified as ``False``, + then the buffer is *not* flushed when the handler is closed. If not specified + or specified as ``True``, the previous behaviour of flushing the buffer will + occur when the handler is closed. .. versionchanged:: 3.6 The *flushOnClose* parameter was added. @@ -959,14 +992,21 @@ possible, while any potentially slow operations (such as sending an email via .. class:: QueueHandler(queue) Returns a new instance of the :class:`QueueHandler` class. The instance is - initialized with the queue to send messages to. The queue can be any - queue-like object; it's used as-is by the :meth:`enqueue` method, which needs - to know how to send messages to it. + initialized with the queue to send messages to. The *queue* can be any + queue-like object; it's used as-is by the :meth:`enqueue` method, which + needs to know how to send messages to it. The queue is not *required* to + have the task tracking API, which means that you can use + :class:`~queue.SimpleQueue` instances for *queue*. .. method:: emit(record) - Enqueues the result of preparing the LogRecord. + Enqueues the result of preparing the LogRecord. Should an exception + occur (e.g. because a bounded queue has filled up), the + :meth:`~logging.Handler.handleError` method is called to handle the + error. This can result in the record silently being dropped (if + :attr:`logging.raiseExceptions` is ``False``) or a message printed to + ``sys.stderr`` (if :attr:`logging.raiseExceptions` is ``True``). .. method:: prepare(record) @@ -1017,14 +1057,17 @@ possible, while any potentially slow operations (such as sending an email via initialized with the queue to send messages to and a list of handlers which will handle entries placed on the queue. The queue can be any queue-like object; it's passed as-is to the :meth:`dequeue` method, which needs - to know how to get messages from it. If ``respect_handler_level`` is ``True``, - a handler's level is respected (compared with the level for the message) when - deciding whether to pass messages to that handler; otherwise, the behaviour - is as in previous Python versions - to always pass each message to each - handler. + to know how to get messages from it. The queue is not *required* to have the + task tracking API (though it's used if available), which means that you can + use :class:`~queue.SimpleQueue` instances for *queue*. + + If ``respect_handler_level`` is ``True``, a handler's level is respected + (compared with the level for the message) when deciding whether to pass + messages to that handler; otherwise, the behaviour is as in previous Python + versions - to always pass each message to each handler. .. versionchanged:: 3.5 - The ``respect_handler_levels`` argument was added. + The ``respect_handler_level`` argument was added. .. method:: dequeue(block) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 08555c3a357602..fb8ea705b0469f 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -50,8 +50,8 @@ listed below. Logger Objects -------------- -Loggers have the following attributes and methods. Note that Loggers are never -instantiated directly, but always through the module-level function +Loggers have the following attributes and methods. Note that Loggers should +*NEVER* be instantiated directly, but always through the module-level function ``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same name will always return a reference to the same Logger object. @@ -124,11 +124,11 @@ is the module's name in the Python package namespace. :meth:`isEnabledFor` will return/expect to be passed integers. - .. method:: Logger.isEnabledFor(lvl) + .. method:: Logger.isEnabledFor(level) - Indicates if a message of severity *lvl* would be processed by this logger. + Indicates if a message of severity *level* would be processed by this logger. This method checks first the module-level level set by - ``logging.disable(lvl)`` and then the logger's effective level as determined + ``logging.disable(level)`` and then the logger's effective level as determined by :meth:`getEffectiveLevel`. @@ -159,6 +159,7 @@ is the module's name in the Python package namespace. message format string, and the *args* are the arguments which are merged into *msg* using the string formatting operator. (Note that this means that you can use keywords in the format string, together with a single dictionary argument.) + No % formatting operation is performed on *msg* when no *args* are supplied. There are four keyword arguments in *kwargs* which are inspected: *exc_info*, *stack_info*, *stacklevel* and *extra*. @@ -269,9 +270,9 @@ is the module's name in the Python package namespace. interpreted as for :meth:`debug`. - .. method:: Logger.log(lvl, msg, *args, **kwargs) + .. method:: Logger.log(level, msg, *args, **kwargs) - Logs a message with integer level *lvl* on this logger. The other arguments are + Logs a message with integer level *level* on this logger. The other arguments are interpreted as for :meth:`debug`. @@ -294,7 +295,7 @@ is the module's name in the Python package namespace. .. method:: Logger.filter(record) - Applies this logger's filters to the record and returns a true value if the + Apply this logger's filters to the record and return ``True`` if the record is to be processed. The filters are consulted in turn, until one of them returns a false value. If none of them return a false value, the record will be processed (passed to handlers). If one returns a false value, no @@ -333,7 +334,7 @@ is the module's name in the Python package namespace. Logger-level filtering is applied using :meth:`~Logger.filter`. - .. method:: Logger.makeRecord(name, lvl, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None) + .. method:: Logger.makeRecord(name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None) This is a factory method which can be overridden in subclasses to create specialized :class:`LogRecord` instances. @@ -447,7 +448,7 @@ subclasses. However, the :meth:`__init__` method in subclasses needs to call .. method:: Handler.filter(record) - Applies this handler's filters to the record and returns a true value if the + Apply this handler's filters to the record and return ``True`` if the record is to be processed. The filters are consulted in turn, until one of them returns a false value. If none of them return a false value, the record will be emitted. If one returns a false value, the handler will not emit the @@ -528,7 +529,7 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on :ref:`logrecord-attributes`. -.. class:: Formatter(fmt=None, datefmt=None, style='%') +.. class:: Formatter(fmt=None, datefmt=None, style='%', validate=True, *, defaults=None) Returns a new instance of the :class:`Formatter` class. The instance is initialized with a format string for the message as a whole, as well as a @@ -538,8 +539,15 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on The *style* parameter can be one of '%', '{' or '$' and determines how the format string will be merged with its data: using one of %-formatting, - :meth:`str.format` or :class:`string.Template`. See :ref:`formatting-styles` - for more information on using {- and $-formatting for log messages. + :meth:`str.format` or :class:`string.Template`. This only applies to the + format string *fmt* (e.g. ``'%(message)s'`` or ``{message}``), not to the + actual log messages passed to ``Logger.debug`` etc; see + :ref:`formatting-styles` for more information on using {- and $-formatting + for log messages. + + The *defaults* parameter can be a dictionary with default values to use in + custom fields. For example: + ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` .. versionchanged:: 3.2 The *style* parameter was added. @@ -549,6 +557,9 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on will raise a ``ValueError``. For example: ``logging.Formatter('%(asctime)s - %(message)s', style='{')``. + .. versionchanged:: 3.10 + The *defaults* parameter was added. + .. method:: format(record) The record's attribute dictionary is used as the operand to a string @@ -563,9 +574,9 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on pickled and sent across the wire, but you should be careful if you have more than one :class:`Formatter` subclass which customizes the formatting of exception information. In this case, you will have to clear the cached - value after a formatter has done its formatting, so that the next - formatter to handle the event doesn't use the cached value but - recalculates it afresh. + value (by setting the *exc_text* attribute to ``None``) after a formatter + has done its formatting, so that the next formatter to handle the event + doesn't use the cached value, but recalculates it afresh. If stack information is available, it's appended after the exception information, using :meth:`formatStack` to transform it if necessary. @@ -604,6 +615,9 @@ The useful mapping keys in a :class:`LogRecord` are given in the section on attributes are ``default_time_format`` (for the strptime format string) and ``default_msec_format`` (for appending the millisecond value). + .. versionchanged:: 3.9 + The ``default_msec_format`` can be ``None``. + .. method:: formatException(exc_info) Formats the specified exception information (a standard exception tuple as @@ -1063,12 +1077,12 @@ functions. handlers being added multiple times to the root logger, which can in turn lead to multiple messages for the same event. -.. function:: disable(lvl=CRITICAL) +.. function:: disable(level=CRITICAL) - Provides an overriding level *lvl* for all loggers which takes precedence over + Provides an overriding level *level* for all loggers which takes precedence over the logger's own level. When the need arises to temporarily throttle logging output down across the whole application, this function can be useful. Its - effect is to disable all logging calls of severity *lvl* and below, so that + effect is to disable all logging calls of severity *level* and below, so that if you call it with a value of INFO, then all INFO and DEBUG events would be discarded, whereas those of severity WARNING and above would be processed according to the logger's effective level. If @@ -1078,16 +1092,16 @@ functions. Note that if you have defined any custom logging level higher than ``CRITICAL`` (this is not recommended), you won't be able to rely on the - default value for the *lvl* parameter, but will have to explicitly supply a + default value for the *level* parameter, but will have to explicitly supply a suitable value. .. versionchanged:: 3.7 - The *lvl* parameter was defaulted to level ``CRITICAL``. See Issue + The *level* parameter was defaulted to level ``CRITICAL``. See Issue #28524 for more information about this change. -.. function:: addLevelName(lvl, levelName) +.. function:: addLevelName(level, levelName) - Associates level *lvl* with text *levelName* in an internal dictionary, which is + Associates level *level* with text *levelName* in an internal dictionary, which is used to map numeric levels to a textual representation, for example when a :class:`Formatter` formats a message. This function can also be used to define your own levels. The only constraints are that all levels used must be @@ -1097,15 +1111,15 @@ functions. .. note:: If you are thinking of defining your own levels, please see the section on :ref:`custom-levels`. -.. function:: getLevelName(lvl) +.. function:: getLevelName(level) - Returns the textual representation of logging level *lvl*. If the level is one + Returns the textual representation of logging level *level*. If the level is one of the predefined levels :const:`CRITICAL`, :const:`ERROR`, :const:`WARNING`, :const:`INFO` or :const:`DEBUG` then you get the corresponding string. If you have associated levels with names using :func:`addLevelName` then the name you - have associated with *lvl* is returned. If a numeric value corresponding to one + have associated with *level* is returned. If a numeric value corresponding to one of the defined levels is passed in, the corresponding string representation is - returned. Otherwise, the string 'Level %s' % lvl is returned. + returned. Otherwise, the string 'Level %s' % level is returned. .. note:: Levels are internally integers (as they need to be compared in the logging logic). This function is used to convert between an integer level @@ -1151,9 +1165,9 @@ functions. +--------------+---------------------------------------------+ | Format | Description | +==============+=============================================+ - | *filename* | Specifies that a FileHandler be created, | - | | using the specified filename, rather than a | - | | StreamHandler. | + | *filename* | Specifies that a :class:`FileHandler` be | + | | created, using the specified filename, | + | | rather than a :class:`StreamHandler`. | +--------------+---------------------------------------------+ | *filemode* | If *filename* is specified, open the file | | | in this :ref:`mode `. Defaults | @@ -1177,9 +1191,10 @@ functions. | | :ref:`level `. | +--------------+---------------------------------------------+ | *stream* | Use the specified stream to initialize the | - | | StreamHandler. Note that this argument is | - | | incompatible with *filename* - if both | - | | are present, a ``ValueError`` is raised. | + | | :class:`StreamHandler`. Note that this | + | | argument is incompatible with *filename* - | + | | if both are present, a ``ValueError`` is | + | | raised. | +--------------+---------------------------------------------+ | *handlers* | If specified, this should be an iterable of | | | already created handlers to add to the root | @@ -1196,6 +1211,21 @@ functions. | | carrying out the configuration as specified | | | by the other arguments. | +--------------+---------------------------------------------+ + | *encoding* | If this keyword argument is specified along | + | | with *filename*, its value is used when the | + | | :class:`FileHandler` is created, and thus | + | | used when opening the output file. | + +--------------+---------------------------------------------+ + | *errors* | If this keyword argument is specified along | + | | with *filename*, its value is used when the | + | | :class:`FileHandler` is created, and thus | + | | used when opening the output file. If not | + | | specified, the value 'backslashreplace' is | + | | used. Note that if ``None`` is specified, | + | | it will be passed as such to :func:`open`, | + | | which means that it will be treated the | + | | same as passing 'errors'. | + +--------------+---------------------------------------------+ .. versionchanged:: 3.2 The *style* argument was added. @@ -1209,6 +1239,9 @@ functions. .. versionchanged:: 3.8 The *force* argument was added. + .. versionchanged:: 3.9 + The *encoding* and *errors* arguments were added. + .. function:: shutdown() Informs the logging system to perform an orderly shutdown by flushing and @@ -1226,7 +1259,9 @@ functions. The class should define :meth:`__init__` such that only a name argument is required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This function is typically called before any loggers are instantiated by applications - which need to use custom logger behavior. + which need to use custom logger behavior. After this call, as at any other + time, do not instantiate loggers directly using the subclass: continue to use + the :func:`logging.getLogger` API to get your loggers. .. function:: setLogRecordFactory(factory) diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index cce6c23e611e31..4bfff9c6147ed4 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -313,7 +313,7 @@ Miscellaneous .. function:: is_check_supported(check) - Returns true if the given integrity check is supported on this system. + Return ``True`` if the given integrity check is supported on this system. :const:`CHECK_NONE` and :const:`CHECK_CRC32` are always supported. :const:`CHECK_CRC64` and :const:`CHECK_SHA256` may be unavailable if you are diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index d901ad2cc1c48e..f82a3b200deb7c 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -308,6 +308,9 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. representation. If *create* is ``True``, the mailbox is created if it does not exist. + If *create* is ``True`` and the *dirname* path exists, it will be treated as + an existing maildir without attempting to verify its directory layout. + It is for historical reasons that *dirname* is named as such rather than *path*. Maildir is a directory-based mailbox format invented for the qmail mail diff --git a/Doc/library/mailcap.rst b/Doc/library/mailcap.rst index 896afd1d730628..bf9639bdaca50a 100644 --- a/Doc/library/mailcap.rst +++ b/Doc/library/mailcap.rst @@ -18,7 +18,7 @@ belonging to a temporary file) and the :program:`xmpeg` program can be automatically started to view the file. The mailcap format is documented in :rfc:`1524`, "A User Agent Configuration -Mechanism For Multimedia Mail Format Information," but is not an Internet +Mechanism For Multimedia Mail Format Information", but is not an Internet standard. However, mailcap files are supported on most Unix systems. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 7129525c788767..f152c45a87aa37 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -10,8 +10,8 @@ -------------- -This module is always available. It provides access to the mathematical -functions defined by the C standard. +This module provides access to the mathematical functions defined by the C +standard. These functions cannot be used with complex numbers; use the functions of the same name from the :mod:`cmath` module if you require support for complex @@ -36,6 +36,24 @@ Number-theoretic and representation functions :class:`~numbers.Integral` value. +.. function:: comb(n, k) + + Return the number of ways to choose *k* items from *n* items without repetition + and without order. + + Evaluates to ``n! / (k! * (n - k)!)`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + Also called the binomial coefficient because it is equivalent + to the coefficient of k-th term in polynomial expansion of the + expression ``(1 + x) ** n``. + + Raises :exc:`TypeError` if either of the arguments are not integers. + Raises :exc:`ValueError` if either of the arguments are negative. + + .. versionadded:: 3.8 + + .. function:: copysign(x, y) Return a float with the magnitude (absolute value) of *x* but the sign of @@ -50,9 +68,12 @@ Number-theoretic and representation functions .. function:: factorial(x) - Return *x* factorial. Raises :exc:`ValueError` if *x* is not integral or + Return *x* factorial as an integer. Raises :exc:`ValueError` if *x* is not integral or is negative. + .. deprecated:: 3.9 + Accepting floats with integral values (like ``5.0``) is deprecated. + .. function:: floor(x) @@ -102,18 +123,23 @@ Number-theoretic and representation functions For further discussion and two alternative approaches, see the `ASPN cookbook recipes for accurate floating point summation - `_\. + `_\. -.. function:: gcd(a, b) +.. function:: gcd(*integers) - Return the greatest common divisor of the integers *a* and *b*. If either - *a* or *b* is nonzero, then the value of ``gcd(a, b)`` is the largest - positive integer that divides both *a* and *b*. ``gcd(0, 0)`` returns - ``0``. + Return the greatest common divisor of the specified integer arguments. + If any of the arguments is nonzero, then the returned value is the largest + positive integer that is a divisor af all arguments. If all arguments + are zero, then the returned value is ``0``. ``gcd()`` without arguments + returns ``0``. .. versionadded:: 3.5 + .. versionchanged:: 3.9 + Added support for an arbitrary number of arguments. Formerly, only two + arguments were supported. + .. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) @@ -166,6 +192,31 @@ Number-theoretic and representation functions Return ``True`` if *x* is a NaN (not a number), and ``False`` otherwise. +.. function:: isqrt(n) + + Return the integer square root of the nonnegative integer *n*. This is the + floor of the exact square root of *n*, or equivalently the greatest integer + *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. + + For some applications, it may be more convenient to have the least integer + *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of + the exact square root of *n*. For positive *n*, this can be computed using + ``a = 1 + isqrt(n - 1)``. + + .. versionadded:: 3.8 + + +.. function:: lcm(*integers) + + Return the least common multiple of the specified integer arguments. + If all arguments are nonzero, then the returned value is the smallest + positive integer that is a multiple of all arguments. If any of the arguments + is zero, then the returned value is ``0``. ``lcm()`` without arguments + returns ``1``. + + .. versionadded:: 3.9 + + .. function:: ldexp(x, i) Return ``x * (2**i)``. This is essentially the inverse of function @@ -178,6 +229,40 @@ Number-theoretic and representation functions of *x* and are floats. +.. function:: nextafter(x, y) + + Return the next floating-point value after *x* towards *y*. + + If *x* is equal to *y*, return *y*. + + Examples: + + * ``math.nextafter(x, math.inf)`` goes up: towards positive infinity. + * ``math.nextafter(x, -math.inf)`` goes down: towards minus infinity. + * ``math.nextafter(x, 0.0)`` goes towards zero. + * ``math.nextafter(x, math.copysign(math.inf, x))`` goes away from zero. + + See also :func:`math.ulp`. + + .. versionadded:: 3.9 + +.. function:: perm(n, k=None) + + Return the number of ways to choose *k* items from *n* items + without repetition and with order. + + Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + If *k* is not specified or is None, then *k* defaults to *n* + and the function returns ``n!``. + + Raises :exc:`TypeError` if either of the arguments are not integers. + Raises :exc:`ValueError` if either of the arguments are negative. + + .. versionadded:: 3.8 + + .. function:: prod(iterable, *, start=1) Calculate the product of all the elements in the input *iterable*. @@ -217,6 +302,30 @@ Number-theoretic and representation functions :class:`~numbers.Integral` (usually an integer). Delegates to :meth:`x.__trunc__() `. +.. function:: ulp(x) + + Return the value of the least significant bit of the float *x*: + + * If *x* is a NaN (not a number), return *x*. + * If *x* is negative, return ``ulp(-x)``. + * If *x* is a positive infinity, return *x*. + * If *x* is equal to zero, return the smallest positive + *denormalized* representable float (smaller than the minimum positive + *normalized* float, :data:`sys.float_info.min `). + * If *x* is equal to the largest positive representable float, + return the value of the least significant bit of *x*, such that the first + float smaller than *x* is ``x - ulp(x)``. + * Otherwise (*x* is a positive finite number), return the value of the least + significant bit of *x*, such that the first float bigger than *x* + is ``x + ulp(x)``. + + ULP stands for "Unit in the Last Place". + + See also :func:`math.nextafter` and :data:`sys.float_info.epsilon + `. + + .. versionadded:: 3.9 + Note that :func:`frexp` and :func:`modf` have a different call/return pattern than their C equivalents: they take a single argument and return a pair of @@ -314,17 +423,20 @@ Trigonometric functions .. function:: acos(x) - Return the arc cosine of *x*, in radians. + Return the arc cosine of *x*, in radians. The result is between ``0`` and + ``pi``. .. function:: asin(x) - Return the arc sine of *x*, in radians. + Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and + ``pi/2``. .. function:: atan(x) - Return the arc tangent of *x*, in radians. + Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and + ``pi/2``. .. function:: atan2(y, x) @@ -345,7 +457,8 @@ Trigonometric functions .. function:: dist(p, q) Return the Euclidean distance between two points *p* and *q*, each - given as a tuple of coordinates. The two tuples must be the same size. + given as a sequence (or iterable) of coordinates. The two points + must have the same dimension. Roughly equivalent to:: @@ -368,6 +481,11 @@ Trigonometric functions Added support for n-dimensional points. Formerly, only the two dimensional case was supported. + .. versionchanged:: 3.10 + Improved the algorithm's accuracy so that the maximum error is + under 1 ulp (unit in the last place). More typically, the result + is almost always correctly rounded to within 1/2 ulp. + .. function:: sin(x) @@ -538,3 +656,6 @@ Constants Module :mod:`cmath` Complex number versions of many of these functions. + +.. |nbsp| unicode:: 0xA0 + :trim: diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 5728407cb34c48..f610032acbe417 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -93,6 +93,10 @@ behavior of the module. Specifying an empty list for *files* will prevent the system defaults from being applied: only the well-known values will be present from a built-in list. + If *files* is ``None`` the internal data structure is completely rebuilt to its + initial default value. This is a stable operation and will produce the same results + when called multiple times. + .. versionchanged:: 3.2 Previously, Windows registry settings were ignored. diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 0f895d76b83fa8..698c17653786b3 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -67,6 +67,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length will be relative to the offset from the beginning of the file. *offset* defaults to 0. *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`. + .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap .. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset]) :noindex: @@ -80,7 +81,9 @@ To map anonymous memory, -1 should be passed as the fileno along with the length private copy-on-write mapping, so changes to the contents of the mmap object will be private to this process, and :const:`MAP_SHARED` creates a mapping that's shared with all other processes mapping the same areas of - the file. The default value is :const:`MAP_SHARED`. + the file. The default value is :const:`MAP_SHARED`. Some systems have + additional possible flags with the full list specified in + :ref:`MAP_* constants `. *prot*, if specified, gives the desired memory protection; the two most useful values are :const:`PROT_READ` and :const:`PROT_WRITE`, to specify @@ -155,6 +158,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length mm.close() + .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap Memory-mapped file objects support the following methods: @@ -201,6 +205,20 @@ To map anonymous memory, -1 should be passed as the fileno along with the length exception was raised on error under Unix. + .. method:: madvise(option[, start[, length]]) + + Send advice *option* to the kernel about the memory region beginning at + *start* and extending *length* bytes. *option* must be one of the + :ref:`MADV_* constants ` available on the system. If + *start* and *length* are omitted, the entire mapping is spanned. On + some systems (including Linux), *start* must be a multiple of the + :const:`PAGESIZE`. + + Availability: Systems with the ``madvise()`` system call. + + .. versionadded:: 3.8 + + .. method:: move(dest, src, count) Copy the *count* bytes starting at offset *src* to the destination index @@ -228,7 +246,8 @@ To map anonymous memory, -1 should be passed as the fileno along with the length .. method:: readline() Returns a single line, starting at the current file position and up to the - next newline. + next newline. The file position is updated to point after the bytes that were + returned. .. method:: resize(newsize) @@ -290,3 +309,56 @@ To map anonymous memory, -1 should be passed as the fileno along with the length position of the file pointer; the file position is advanced by ``1``. If the mmap was created with :const:`ACCESS_READ`, then writing to it will raise a :exc:`TypeError` exception. + +.. _madvise-constants: + +MADV_* Constants +++++++++++++++++ + +.. data:: MADV_NORMAL + MADV_RANDOM + MADV_SEQUENTIAL + MADV_WILLNEED + MADV_DONTNEED + MADV_REMOVE + MADV_DONTFORK + MADV_DOFORK + MADV_HWPOISON + MADV_MERGEABLE + MADV_UNMERGEABLE + MADV_SOFT_OFFLINE + MADV_HUGEPAGE + MADV_NOHUGEPAGE + MADV_DONTDUMP + MADV_DODUMP + MADV_FREE + MADV_NOSYNC + MADV_AUTOSYNC + MADV_NOCORE + MADV_CORE + MADV_PROTECT + + These options can be passed to :meth:`mmap.madvise`. Not every option will + be present on every system. + + Availability: Systems with the madvise() system call. + + .. versionadded:: 3.8 + +.. _map-constants: + +MAP_* Constants ++++++++++++++++ + +.. data:: MAP_SHARED + MAP_PRIVATE + MAP_DENYWRITE + MAP_EXECUTABLE + MAP_ANON + MAP_ANONYMOUS + MAP_POPULATE + + These are the various flags that can be passed to :meth:`mmap.mmap`. Note that some options might not be present on some systems. + + .. versionchanged:: 3.10 + Added MAP_POPULATE constant. diff --git a/Doc/library/modules.rst b/Doc/library/modules.rst index 6b2a40a1b71476..565ce0525c2c99 100644 --- a/Doc/library/modules.rst +++ b/Doc/library/modules.rst @@ -17,3 +17,4 @@ The full list of modules described in this chapter is: modulefinder.rst runpy.rst importlib.rst + importlib.metadata.rst diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index bd34ffb1b2335d..42fffee6a0f449 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -42,6 +42,8 @@ File Operations regions in a file may be locked at the same time, but may not overlap. Adjacent regions are not merged; they must be unlocked individually. + .. audit-event:: msvcrt.locking fd,mode,nbytes msvcrt.locking + .. data:: LK_LOCK LK_RLCK @@ -77,12 +79,16 @@ File Operations and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter to :func:`os.fdopen` to create a file object. + .. audit-event:: msvcrt.open_osfhandle handle,flags msvcrt.open_osfhandle + .. function:: get_osfhandle(fd) Return the file handle for the file descriptor *fd*. Raises :exc:`OSError` if *fd* is not recognized. + .. audit-event:: msvcrt.get_osfhandle fd msvcrt.get_osfhandle + .. _msvcrt-console: @@ -92,7 +98,7 @@ Console I/O .. function:: kbhit() - Return true if a keypress is waiting to be read. + Return ``True`` if a keypress is waiting to be read. .. function:: getch() diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index a5ecfa6cc1c703..f8e7556f25b4e0 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -14,7 +14,8 @@ Introduction :mod:`multiprocessing` is a package that supports spawning processes using an API similar to the :mod:`threading` module. The :mod:`multiprocessing` package offers both local and remote concurrency, effectively side-stepping the -:term:`Global Interpreter Lock` by using subprocesses instead of threads. Due +:term:`Global Interpreter Lock ` by using +subprocesses instead of threads. Due to this, the :mod:`multiprocessing` module allows the programmer to fully leverage multiple processors on a given machine. It runs on both Unix and Windows. @@ -102,7 +103,7 @@ to start a process. These *start methods* are will not be inherited. Starting a process using this method is rather slow compared to using *fork* or *forkserver*. - Available on Unix and Windows. The default on Windows. + Available on Unix and Windows. The default on Windows and macOS. *fork* The parent process uses :func:`os.fork` to fork the Python @@ -124,6 +125,12 @@ to start a process. These *start methods* are Available on Unix platforms which support passing file descriptors over Unix pipes. +.. versionchanged:: 3.8 + + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess. See :issue:`33725`. + .. versionchanged:: 3.4 *spawn* added on all unix platforms, and *forkserver* added for some unix platforms. @@ -131,13 +138,17 @@ to start a process. These *start methods* are handles on Windows. On Unix using the *spawn* or *forkserver* start methods will also -start a *semaphore tracker* process which tracks the unlinked named -semaphores created by processes of the program. When all processes -have exited the semaphore tracker unlinks any remaining semaphores. +start a *resource tracker* process which tracks the unlinked named +system resources (such as named semaphores or +:class:`~multiprocessing.shared_memory.SharedMemory` objects) created +by processes of the program. When all processes +have exited the resource tracker unlinks any remaining tracked object. Usually there should be none, but if a process was killed by a signal -there may be some "leaked" semaphores. (Unlinking the named semaphores -is a serious matter since the system allows only a limited number, and -they will not be automatically unlinked until the next reboot.) +there may be some "leaked" resources. (Neither leaked semaphores nor shared +memory segments will be automatically unlinked until the next reboot. This is +problematic for both objects because the system allows only a limited number of +named semaphores, and shared memory segments occupy some space in the main +memory.) To select a start method you use the :func:`set_start_method` in the ``if __name__ == '__main__'`` clause of the main module. For @@ -429,7 +440,8 @@ process which created it. >>> def f(x): ... return x*x ... - >>> p.map(f, [1,2,3]) + >>> with p: + ... p.map(f, [1,2,3]) Process PoolWorker-1: Process PoolWorker-2: Process PoolWorker-3: @@ -849,7 +861,7 @@ For an example of the usage of queues for interprocess communication see A better name for this method might be ``allow_exit_without_flush()``. It is likely to cause enqueued - data to lost, and you almost certainly will not need to use it. + data to be lost, and you almost certainly will not need to use it. It is really only there if you need the current process to exit immediately without waiting to flush enqueued data to the underlying pipe, and you don't care about lost data. @@ -867,6 +879,16 @@ For an example of the usage of queues for interprocess communication see It is a simplified :class:`Queue` type, very close to a locked :class:`Pipe`. + .. method:: close() + + Close the queue: release internal resources. + + A queue must not be used anymore after it is closed. For example, + :meth:`get`, :meth:`put` and :meth:`empty` methods must no longer be + called. + + .. versionadded:: 3.9 + .. method:: empty() Return ``True`` if the queue is empty, ``False`` otherwise. @@ -940,6 +962,14 @@ Miscellaneous An analogue of :func:`threading.current_thread`. +.. function:: parent_process() + + Return the :class:`Process` object corresponding to the parent process of + the :func:`current_process`. For the main process, ``parent_process`` will + be ``None``. + + .. versionadded:: 3.8 + .. function:: freeze_support() Add support for when a program which uses :mod:`multiprocessing` has been @@ -2109,6 +2139,16 @@ with the :class:`Pool` class. Note that the methods of the pool object should only be called by the process which created the pool. + .. warning:: + :class:`multiprocessing.pool` objects have internal resources that need to be + properly managed (like any other resource) by using the pool as a context manager + or by calling :meth:`close` and :meth:`terminate` manually. Failure to do this + can lead to the process hanging on finalization. + + Note that it is **not correct** to rely on the garbage collector to destroy the pool + as CPython does not assure that the finalizer of the pool will be called + (see :meth:`object.__del__` for more information). + .. versionadded:: 3.2 *maxtasksperchild* @@ -2134,7 +2174,8 @@ with the :class:`Pool` class. .. method:: apply_async(func[, args[, kwds[, callback[, error_callback]]]]) - A variant of the :meth:`apply` method which returns a result object. + A variant of the :meth:`apply` method which returns a + :class:`~multiprocessing.pool.AsyncResult` object. If *callback* is specified then it should be a callable which accepts a single argument. When the result becomes ready *callback* is applied to @@ -2151,7 +2192,8 @@ with the :class:`Pool` class. .. method:: map(func, iterable[, chunksize]) A parallel equivalent of the :func:`map` built-in function (it supports only - one *iterable* argument though). It blocks until the result is ready. + one *iterable* argument though, for multiple iterables see :meth:`starmap`). + It blocks until the result is ready. This method chops the iterable into a number of chunks which it submits to the process pool as separate tasks. The (approximate) size of these @@ -2163,7 +2205,8 @@ with the :class:`Pool` class. .. method:: map_async(func, iterable[, chunksize[, callback[, error_callback]]]) - A variant of the :meth:`.map` method which returns a result object. + A variant of the :meth:`.map` method which returns a + :class:`~multiprocessing.pool.AsyncResult` object. If *callback* is specified then it should be a callable which accepts a single argument. When the result becomes ready *callback* is applied to @@ -2260,7 +2303,11 @@ with the :class:`Pool` class. .. method:: successful() Return whether the call completed without raising an exception. Will - raise :exc:`AssertionError` if the result is not ready. + raise :exc:`ValueError` if the result is not ready. + + .. versionchanged:: 3.7 + If the result is not ready, :exc:`ValueError` is raised instead of + :exc:`AssertionError`. The following example demonstrates the use of a pool:: diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst index 56188c7ef53880..e7ec9047e015e6 100644 --- a/Doc/library/nntplib.rst +++ b/Doc/library/nntplib.rst @@ -20,7 +20,7 @@ as well as the older :rfc:`977` and :rfc:`2980`. Here are two small examples of how it can be used. To list some statistics about a newsgroup and print the subjects of the last 10 articles:: - >>> s = nntplib.NNTP('news.gmane.org') + >>> s = nntplib.NNTP('news.gmane.io') >>> resp, count, first, last, name = s.group('gmane.comp.python.committers') >>> print('Group', name, 'has', count, 'articles, range', first, 'to', last) Group gmane.comp.python.committers has 1096 articles, range 1 to 1096 @@ -44,7 +44,7 @@ about a newsgroup and print the subjects of the last 10 articles:: To post an article from a binary file (this assumes that the article has valid headers, and that you have right to post on the particular newsgroup):: - >>> s = nntplib.NNTP('news.gmane.org') + >>> s = nntplib.NNTP('news.gmane.io') >>> f = open('article.txt', 'rb') >>> s.post(f) '240 Article posted successfully.' @@ -73,12 +73,19 @@ The module itself defines the following classes: connection when done, e.g.: >>> from nntplib import NNTP - >>> with NNTP('news.gmane.org') as n: + >>> with NNTP('news.gmane.io') as n: ... n.group('gmane.comp.python.committers') ... # doctest: +SKIP ('211 1755 1 1755 gmane.comp.python.committers', 1755, 1, 1755, 'gmane.comp.python.committers') >>> + .. audit-event:: nntplib.connect self,host,port nntplib.NNTP + + .. audit-event:: nntplib.putline self,line nntplib.NNTP + + All commands will raise an :ref:`auditing event ` + ``nntplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. .. versionchanged:: 3.2 *usenetrc* is now ``False`` by default. @@ -86,6 +93,10 @@ The module itself defines the following classes: .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. + .. versionchanged:: 3.9 + If the *timeout* parameter is set to be zero, it will raise a + :class:`ValueError` to prevent the creation of a non-blocking socket. + .. class:: NNTP_SSL(host, port=563, user=None, password=None, ssl_context=None, readermode=None, usenetrc=False, [timeout]) Return a new :class:`NNTP_SSL` object, representing an encrypted @@ -100,6 +111,14 @@ The module itself defines the following classes: STARTTLS as described below. However, some servers only support the former. + .. audit-event:: nntplib.connect self,host,port nntplib.NNTP_SSL + + .. audit-event:: nntplib.putline self,line nntplib.NNTP_SSL + + All commands will raise an :ref:`auditing event ` + ``nntplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. + .. versionadded:: 3.2 .. versionchanged:: 3.4 @@ -107,6 +126,10 @@ The module itself defines the following classes: :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see :data:`ssl.HAS_SNI`). + .. versionchanged:: 3.9 + If the *timeout* parameter is set to be zero, it will raise a + :class:`ValueError` to prevent the creation of a non-blocking socket. + .. exception:: NNTPError Derived from the standard exception :exc:`Exception`, this is the base @@ -210,7 +233,7 @@ tuples or objects that the method normally returns will be empty. of values. On legacy servers which don't understand the ``CAPABILITIES`` command, an empty dictionary is returned instead. - >>> s = NNTP('news.gmane.org') + >>> s = NNTP('news.gmane.io') >>> 'POST' in s.getcapabilities() True @@ -519,33 +542,6 @@ them have been superseded by newer commands in :rfc:`3977`. if available. -.. method:: NNTP.xpath(id) - - Return a pair ``(resp, path)``, where *path* is the directory path to the - article with message ID *id*. Most of the time, this extension is not - enabled by NNTP server administrators. - - .. deprecated:: 3.3 - The XPATH extension is not actively used. - - -.. XXX deprecated: - - .. method:: NNTP.xgtitle(name, *, file=None) - - Process an ``XGTITLE`` command, returning a pair ``(response, list)``, where - *list* is a list of tuples containing ``(name, title)``. If the *file* parameter - is supplied, then the output of the ``XGTITLE`` command is stored in a file. - If *file* is a string, then the method will open a file with that name, write - to it then close it. If *file* is a :term:`file object`, then it will start - calling :meth:`write` on it to store the lines of the command output. If *file* - is supplied, then the returned *list* is an empty list. This is an optional NNTP - extension, and may not be supported by all servers. - - :rfc:`2980` says "It is suggested that this extension be deprecated". Use - :meth:`descriptions` or :meth:`description` instead. - - Utility functions ----------------- diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 5d0ea7dfdd8928..36c53556c2685e 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -112,6 +112,10 @@ The mathematical and bitwise operations are the most numerous: Return *a* converted to an integer. Equivalent to ``a.__index__()``. + .. versionchanged:: 3.10 + The result always has exact type :class:`int`. Previously, the result + could have been an instance of a subclass of ``int``. + .. function:: inv(obj) invert(obj) @@ -339,7 +343,7 @@ expect a function argument. [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)] -.. function:: methodcaller(name[, args...]) +.. function:: methodcaller(name, /, *args, **kwargs) Return a callable object that calls the method *name* on its operand. If additional arguments and/or keyword arguments are given, they will be given @@ -352,7 +356,7 @@ expect a function argument. Equivalent to:: - def methodcaller(name, *args, **kwargs): + def methodcaller(name, /, *args, **kwargs): def caller(obj): return getattr(obj, name)(*args, **kwargs) return caller diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 16f1cf008fff3e..c1a18e01474319 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -928,10 +928,10 @@ The canonical way to create an :class:`Option` instance is with the store a constant value ``"store_true"`` - store a true value + store ``True`` ``"store_false"`` - store a false value + store ``False`` ``"append"`` append this option's argument to a list @@ -1135,12 +1135,12 @@ must specify for any option using that action. * ``"store_true"`` [relevant: :attr:`~Option.dest`] - A special case of ``"store_const"`` that stores a true value to + A special case of ``"store_const"`` that stores ``True`` to :attr:`~Option.dest`. * ``"store_false"`` [relevant: :attr:`~Option.dest`] - Like ``"store_true"``, but stores a false value. + Like ``"store_true"``, but stores ``False``. Example:: @@ -1396,7 +1396,7 @@ provides several methods to help you out: .. method:: OptionParser.has_option(opt_str) - Return true if the OptionParser has an option with option string *opt_str* + Return ``True`` if the OptionParser has an option with option string *opt_str* (e.g., ``-q`` or ``--verbose``). .. method:: OptionParser.remove_option(opt_str) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index a167e3b885fdfc..a5abacf02144a7 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -87,9 +87,10 @@ the :mod:`glob` module.) .. function:: commonpath(paths) Return the longest common sub-path of each pathname in the sequence - *paths*. Raise ValueError if *paths* contains both absolute and relative - pathnames, or if *paths* is empty. Unlike :func:`commonprefix`, this - returns a valid path. + *paths*. Raise :exc:`ValueError` if *paths* contain both absolute + and relative pathnames, the *paths* are on the different drives or + if *paths* is empty. Unlike :func:`commonprefix`, this returns a + valid path. .. availability:: Unix, Windows. @@ -324,11 +325,9 @@ the :mod:`glob` module.) .. function:: normcase(path) - Normalize the case of a pathname. On Unix and Mac OS X, this returns the - path unchanged; on case-insensitive filesystems, it converts the path to - lowercase. On Windows, it also converts forward slashes to backward slashes. - Raise a :exc:`TypeError` if the type of *path* is not ``str`` or ``bytes`` (directly - or indirectly through the :class:`os.PathLike` interface). + Normalize the case of a pathname. On Windows, convert all characters in the + pathname to lowercase, and also convert forward slashes to backward slashes. + On other operating systems, return the path unchanged. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -349,11 +348,19 @@ the :mod:`glob` module.) .. function:: realpath(path) Return the canonical path of the specified filename, eliminating any symbolic - links encountered in the path (if they are supported by the operating system). + links encountered in the path (if they are supported by the operating + system). + + .. note:: + When symbolic link cycles occur, the returned path will be one member of + the cycle, but no guarantee is made about which member that will be. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.8 + Symbolic links and junctions are now resolved on Windows. + .. function:: relpath(path, start=os.curdir) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index f3b5d964ac58a8..8c3bc5fb87d61b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -32,12 +32,13 @@ Notes on the availability of these functions: objects, and result in an object of the same type, if a path or file name is returned. +* On VxWorks, os.fork, os.execv and os.spawn*p* are not supported. .. note:: - All functions in this module raise :exc:`OSError` in the case of invalid or - inaccessible file names and paths, or other arguments that have the correct - type, but are not accepted by the operating system. + All functions in this module raise :exc:`OSError` (or subclasses thereof) in + the case of invalid or inaccessible file names and paths, or other arguments + that have the correct type, but are not accepted by the operating system. .. exception:: error @@ -110,9 +111,9 @@ process and user. to the environment made after this time are not reflected in ``os.environ``, except for changes made by modifying ``os.environ`` directly. - If the platform supports the :func:`putenv` function, this mapping may be used - to modify the environment as well as query the environment. :func:`putenv` will - be called automatically when the mapping is modified. + This mapping may be used to modify the environment as well as query the + environment. :func:`putenv` will be called automatically when the mapping + is modified. On Unix, keys and values use :func:`sys.getfilesystemencoding` and ``'surrogateescape'`` error handler. Use :data:`environb` if you would like @@ -129,14 +130,13 @@ process and user. cause memory leaks. Refer to the system documentation for :c:func:`putenv`. - If :func:`putenv` is not provided, a modified copy of this mapping may be - passed to the appropriate process-creation functions to cause child processes - to use a modified environment. + You can delete items in this mapping to unset environment variables. + :func:`unsetenv` will be called automatically when an item is deleted from + ``os.environ``, and when one of the :meth:`pop` or :meth:`clear` methods is + called. - If the platform supports the :func:`unsetenv` function, you can delete items in - this mapping to unset environment variables. :func:`unsetenv` will be called - automatically when an item is deleted from ``os.environ``, and when - one of the :meth:`pop` or :meth:`clear` methods is called. + .. versionchanged:: 3.9 + Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. .. data:: environb @@ -147,10 +147,13 @@ process and user. versa). :data:`environb` is only available if :data:`supports_bytes_environ` is - True. + ``True``. .. versionadded:: 3.2 + .. versionchanged:: 3.9 + Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. + .. function:: chdir(path) fchdir(fd) @@ -235,7 +238,7 @@ process and user. *default* if it doesn't. *key*, *default* and the result are bytes. :func:`getenvb` is only available if :data:`supports_bytes_environ` - is True. + is ``True``. .. availability:: most flavors of Unix. @@ -438,17 +441,20 @@ process and user. changes to the environment affect subprocesses started with :func:`os.system`, :func:`popen` or :func:`fork` and :func:`execv`. - .. availability:: most flavors of Unix, Windows. + Assignments to items in ``os.environ`` are automatically translated into + corresponding calls to :func:`putenv`; however, calls to :func:`putenv` + don't update ``os.environ``, so it is actually preferable to assign to items + of ``os.environ``. .. note:: On some platforms, including FreeBSD and Mac OS X, setting ``environ`` may - cause memory leaks. Refer to the system documentation for putenv. + cause memory leaks. Refer to the system documentation for :c:func:`putenv`. + + .. audit-event:: os.putenv key,value os.putenv - When :func:`putenv` is supported, assignments to items in ``os.environ`` are - automatically translated into corresponding calls to :func:`putenv`; however, - calls to :func:`putenv` don't update ``os.environ``, so it is actually - preferable to assign to items of ``os.environ``. + .. versionchanged:: 3.9 + The function is now always available. .. function:: setegid(egid) @@ -637,12 +643,15 @@ process and user. environment affect subprocesses started with :func:`os.system`, :func:`popen` or :func:`fork` and :func:`execv`. - When :func:`unsetenv` is supported, deletion of items in ``os.environ`` is - automatically translated into a corresponding call to :func:`unsetenv`; however, - calls to :func:`unsetenv` don't update ``os.environ``, so it is actually - preferable to delete items of ``os.environ``. + Deletion of items in ``os.environ`` is automatically translated into a + corresponding call to :func:`unsetenv`; however, calls to :func:`unsetenv` + don't update ``os.environ``, so it is actually preferable to delete items of + ``os.environ``. - .. availability:: most flavors of Unix, Windows. + .. audit-event:: os.unsetenv key os.unsetenv + + .. versionchanged:: 3.9 + The function is now always available and is also available on Windows. .. _os-newstreams: @@ -650,7 +659,7 @@ process and user. File Object Creation -------------------- -This function creates new :term:`file objects `. (See also +These functions create new :term:`file objects `. (See also :func:`~os.open` for opening file descriptors.) @@ -706,6 +715,28 @@ as internal buffering of data. pass +.. function:: copy_file_range(src, dst, count, offset_src=None, offset_dst=None) + + Copy *count* bytes from file descriptor *src*, starting from offset + *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. + If *offset_src* is None, then *src* is read from the current position; + respectively for *offset_dst*. The files pointed by *src* and *dst* + must reside in the same filesystem, otherwise an :exc:`OSError` is + raised with :attr:`~OSError.errno` set to :data:`errno.EXDEV`. + + This copy is done without the additional cost of transferring data + from the kernel to user space and then back into the kernel. Additionally, + some filesystems could implement extra optimizations. The copy is done as if + both files are opened as binary. + + The return value is the amount of bytes copied. This could be less than the + amount requested. + + .. availability:: Linux kernel >= 4.5 or glibc >= 2.27. + + .. versionadded:: 3.8 + + .. function:: device_encoding(fd) Return a string describing the encoding of the device associated with *fd* @@ -745,6 +776,8 @@ as internal buffering of data. docs for :func:`chmod` for possible values of *mode*. As of Python 3.3, this is equivalent to ``os.chmod(fd, mode)``. + .. audit-event:: os.chmod path,mode,dir_fd os.fchmod + .. availability:: Unix. @@ -755,6 +788,8 @@ as internal buffering of data. :func:`chown`. As of Python 3.3, this is equivalent to ``os.chown(fd, uid, gid)``. + .. audit-event:: os.chown path,uid,gid,dir_fd os.fchown + .. availability:: Unix. @@ -828,11 +863,14 @@ as internal buffering of data. most *length* bytes in size. As of Python 3.3, this is equivalent to ``os.truncate(fd, length)``. + .. audit-event:: os.truncate fd,length os.ftruncate + .. availability:: Unix, Windows. .. versionchanged:: 3.5 Added support for Windows + .. function:: get_blocking(fd) Get the blocking mode of the file descriptor: ``False`` if the @@ -844,6 +882,7 @@ as internal buffering of data. .. versionadded:: 3.5 + .. function:: isatty(fd) Return ``True`` if the file descriptor *fd* is open and connected to a @@ -858,6 +897,8 @@ as internal buffering of data. :data:`F_ULOCK` or :data:`F_TEST`. *len* specifies the section of the file to lock. + .. audit-event:: os.lockf fd,cmd,len os.lockf + .. availability:: Unix. .. versionadded:: 3.3 @@ -911,6 +952,8 @@ as internal buffering of data. This function can support :ref:`paths relative to directory descriptors ` with the *dir_fd* parameter. + .. audit-event:: open path,mode,flags os.open + .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. @@ -1108,7 +1151,8 @@ or `the MSDN `_ on Windo Combine the functionality of :func:`os.readv` and :func:`os.pread`. .. availability:: Linux 2.6.30 and newer, FreeBSD 6.0 and newer, - OpenBSD 2.7 and newer. Using flags requires Linux 4.6 or newer. + OpenBSD 2.7 and newer, AIX 7.1 and newer. Using flags requires + Linux 4.6 or newer. .. versionadded:: 3.7 @@ -1167,6 +1211,7 @@ or `the MSDN `_ on Windo - :data:`RWF_DSYNC` - :data:`RWF_SYNC` + - :data:`RWF_APPEND` Return the total number of bytes actually written. @@ -1176,15 +1221,16 @@ or `the MSDN `_ on Windo Combine the functionality of :func:`os.writev` and :func:`os.pwrite`. .. availability:: Linux 2.6.30 and newer, FreeBSD 6.0 and newer, - OpenBSD 2.7 and newer. Using flags requires Linux 4.7 or newer. + OpenBSD 2.7 and newer, AIX 7.1 and newer. Using flags requires + Linux 4.7 or newer. .. versionadded:: 3.7 .. data:: RWF_DSYNC - Provide a per-write equivalent of the :data:`O_DSYNC` ``open(2)`` flag. This - flag effect applies only to the data range written by the system call. + Provide a per-write equivalent of the :data:`O_DSYNC` :func:`os.open` flag. + This flag effect applies only to the data range written by the system call. .. availability:: Linux 4.7 and newer. @@ -1193,14 +1239,28 @@ or `the MSDN `_ on Windo .. data:: RWF_SYNC - Provide a per-write equivalent of the :data:`O_SYNC` ``open(2)`` flag. This - flag effect applies only to the data range written by the system call. + Provide a per-write equivalent of the :data:`O_SYNC` :func:`os.open` flag. + This flag effect applies only to the data range written by the system call. .. availability:: Linux 4.7 and newer. .. versionadded:: 3.7 +.. data:: RWF_APPEND + + Provide a per-write equivalent of the :data:`O_APPEND` :func:`os.open` + flag. This flag is meaningful only for :func:`os.pwritev`, and its + effect applies only to the data range written by the system call. The + *offset* argument does not affect the write operation; the data is always + appended to the end of the file. However, if the *offset* argument is + ``-1``, the current file *offset* is updated. + + .. availability:: Linux 4.16 and newer. + + .. versionadded:: 3.10 + + .. function:: read(fd, n) Read at most *n* bytes from file descriptor *fd*. @@ -1222,27 +1282,27 @@ or `the MSDN `_ on Windo :exc:`InterruptedError` exception (see :pep:`475` for the rationale). -.. function:: sendfile(out, in, offset, count) - sendfile(out, in, offset, count, [headers], [trailers], flags=0) +.. function:: sendfile(out_fd, in_fd, offset, count) + sendfile(out_fd, in_fd, offset, count, headers=(), trailers=(), flags=0) - Copy *count* bytes from file descriptor *in* to file descriptor *out* + Copy *count* bytes from file descriptor *in_fd* to file descriptor *out_fd* starting at *offset*. - Return the number of bytes sent. When EOF is reached return 0. + Return the number of bytes sent. When EOF is reached return ``0``. The first function notation is supported by all platforms that define :func:`sendfile`. On Linux, if *offset* is given as ``None``, the bytes are read from the - current position of *in* and the position of *in* is updated. + current position of *in_fd* and the position of *in_fd* is updated. The second case may be used on Mac OS X and FreeBSD where *headers* and *trailers* are arbitrary sequences of buffers that are written before and - after the data from *in* is written. It returns the same as the first case. + after the data from *in_fd* is written. It returns the same as the first case. - On Mac OS X and FreeBSD, a value of 0 for *count* specifies to send until - the end of *in* is reached. + On Mac OS X and FreeBSD, a value of ``0`` for *count* specifies to send until + the end of *in_fd* is reached. - All platforms support sockets as *out* file descriptor, and some platforms + All platforms support sockets as *out_fd* file descriptor, and some platforms allow other types (e.g. regular file, pipe) as well. Cross-platform applications should not use *headers*, *trailers* and *flags* @@ -1257,6 +1317,9 @@ or `the MSDN `_ on Windo .. versionadded:: 3.3 + .. versionchanged:: 3.9 + Parameters *out* and *in* was renamed to *out_fd* and *in_fd*. + .. function:: set_blocking(fd, blocking) @@ -1453,16 +1516,19 @@ features: .. _path_fd: * **specifying a file descriptor:** - For some functions, the *path* argument can be not only a string giving a path - name, but also a file descriptor. The function will then operate on the file - referred to by the descriptor. (For POSIX systems, Python will call the - ``f...`` version of the function.) - - You can check whether or not *path* can be specified as a file descriptor on - your platform using :data:`os.supports_fd`. If it is unavailable, using it - will raise a :exc:`NotImplementedError`. + Normally the *path* argument provided to functions in the :mod:`os` module + must be a string specifying a file path. However, some functions now + alternatively accept an open file descriptor for their *path* argument. + The function will then operate on the file referred to by the descriptor. + (For POSIX systems, Python will call the variant of the function prefixed + with ``f`` (e.g. call ``fchdir`` instead of ``chdir``).) + + You can check whether or not *path* can be specified as a file descriptor + for a particular function on your platform using :data:`os.supports_fd`. + If this functionality is unavailable, using it will raise a + :exc:`NotImplementedError`. - If the function also supports *dir_fd* or *follow_symlinks* arguments, it is + If the function also supports *dir_fd* or *follow_symlinks* arguments, it's an error to specify one of those when supplying *path* as a file descriptor. .. _dir_fd: @@ -1471,23 +1537,24 @@ features: should be a file descriptor referring to a directory, and the path to operate on should be relative; path will then be relative to that directory. If the path is absolute, *dir_fd* is ignored. (For POSIX systems, Python will call - the ``...at`` or ``f...at`` version of the function.) + the variant of the function with an ``at`` suffix and possibly prefixed with + ``f`` (e.g. call ``faccessat`` instead of ``access``). - You can check whether or not *dir_fd* is supported on your platform using - :data:`os.supports_dir_fd`. If it is unavailable, using it will raise a - :exc:`NotImplementedError`. + You can check whether or not *dir_fd* is supported for a particular function + on your platform using :data:`os.supports_dir_fd`. If it's unavailable, + using it will raise a :exc:`NotImplementedError`. .. _follow_symlinks: * **not following symlinks:** If *follow_symlinks* is ``False``, and the last element of the path to operate on is a symbolic link, - the function will operate on the symbolic link itself instead of the file the - link points to. (For POSIX systems, Python will call the ``l...`` version of - the function.) + the function will operate on the symbolic link itself rather than the file + pointed to by the link. (For POSIX systems, Python will call the ``l...`` + variant of the function.) - You can check whether or not *follow_symlinks* is supported on your platform - using :data:`os.supports_follow_symlinks`. If it is unavailable, using it - will raise a :exc:`NotImplementedError`. + You can check whether or not *follow_symlinks* is supported for a particular + function on your platform using :data:`os.supports_follow_symlinks`. + If it's unavailable, using it will raise a :exc:`NotImplementedError`. @@ -1566,6 +1633,11 @@ features: This function can support :ref:`specifying a file descriptor `. The descriptor must refer to an opened directory, not an open file. + This function can raise :exc:`OSError` and subclasses such as + :exc:`FileNotFoundError`, :exc:`PermissionError`, and :exc:`NotADirectoryError`. + + .. audit-event:: os.chdir path os.chdir + .. versionadded:: 3.3 Added support for specifying *path* as a file descriptor on some platforms. @@ -1594,6 +1666,8 @@ features: This function can support :ref:`not following symlinks `. + .. audit-event:: os.chflags path,flags os.chflags + .. availability:: Unix. .. versionadded:: 3.3 @@ -1639,6 +1713,8 @@ features: read-only flag with it (via the ``stat.S_IWRITE`` and ``stat.S_IREAD`` constants or a corresponding integer value). All other bits are ignored. + .. audit-event:: os.chmod path,mode,dir_fd os.chmod + .. versionadded:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. @@ -1659,10 +1735,12 @@ features: See :func:`shutil.chown` for a higher-level function that accepts names in addition to numeric ids. + .. audit-event:: os.chown path,uid,gid,dir_fd os.chown + .. availability:: Unix. .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*, + Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. .. versionchanged:: 3.6 @@ -1685,6 +1763,8 @@ features: descriptor *fd*. The descriptor must refer to an opened directory, not an open file. As of Python 3.3, this is equivalent to ``os.chdir(fd)``. + .. audit-event:: os.chdir path os.fchdir + .. availability:: Unix. @@ -1697,6 +1777,11 @@ features: Return a bytestring representing the current working directory. + .. versionchanged:: 3.8 + The function now uses the UTF-8 encoding on Windows, rather than the ANSI + code page: see :pep:`529` for the rationale. The function is no longer + deprecated on Windows. + .. function:: lchflags(path, flags) @@ -1704,6 +1789,8 @@ features: not follow symbolic links. As of Python 3.3, this is equivalent to ``os.chflags(path, flags, follow_symlinks=False)``. + .. audit-event:: os.chflags path,flags os.lchflags + .. availability:: Unix. .. versionchanged:: 3.6 @@ -1717,6 +1804,8 @@ features: for possible values of *mode*. As of Python 3.3, this is equivalent to ``os.chmod(path, mode, follow_symlinks=False)``. + .. audit-event:: os.chmod path,mode,dir_fd os.lchmod + .. availability:: Unix. .. versionchanged:: 3.6 @@ -1728,6 +1817,8 @@ features: function will not follow symbolic links. As of Python 3.3, this is equivalent to ``os.chown(path, uid, gid, follow_symlinks=False)``. + .. audit-event:: os.chown path,uid,gid,dir_fd os.lchown + .. availability:: Unix. .. versionchanged:: 3.6 @@ -1742,6 +1833,8 @@ features: supply :ref:`paths relative to directory descriptors `, and :ref:`not following symlinks `. + .. audit-event:: os.link src,dst,src_dir_fd,dst_dir_fd os.link + .. availability:: Unix, Windows. .. versionchanged:: 3.2 @@ -1759,6 +1852,8 @@ features: Return a list containing the names of the entries in the directory given by *path*. The list is in arbitrary order, and does not include the special entries ``'.'`` and ``'..'`` even if they are present in the directory. + If a file is removed from or added to the directory during the call of + this function, whether a name for that file be included is unspecified. *path* may be a :term:`path-like object`. If *path* is of type ``bytes`` (directly or indirectly through the :class:`PathLike` interface), @@ -1768,6 +1863,8 @@ features: This function can also support :ref:`specifying a file descriptor `; the file descriptor must refer to a directory. + .. audit-event:: os.listdir path os.listdir + .. note:: To encode ``str`` filenames to ``bytes``, use :func:`~os.fsencode`. @@ -1781,7 +1878,7 @@ features: The *path* parameter became optional. .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*. + Added support for specifying *path* as an open file descriptor. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -1815,6 +1912,12 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *src* and *dst*. + .. versionchanged:: 3.8 + On Windows, now opens reparse points that represent another path + (name surrogates), including symbolic links and directory junctions. + Other kinds of reparse points are resolved by the operating system as + for :func:`~os.stat`. + .. function:: mkdir(path, mode=0o777, *, dir_fd=None) @@ -1836,6 +1939,8 @@ features: It is also possible to create temporary directories; see the :mod:`tempfile` module's :func:`tempfile.mkdtemp` function. + .. audit-event:: os.mkdir path,mode,dir_fd os.mkdir + .. versionadded:: 3.3 The *dir_fd* argument. @@ -1858,8 +1963,8 @@ features: directories you can set the umask before invoking :func:`makedirs`. The file permission bits of existing parent directories are not changed. - If *exist_ok* is ``False`` (the default), an :exc:`OSError` is raised if the - target directory already exists. + If *exist_ok* is ``False`` (the default), an :exc:`FileExistsError` is + raised if the target directory already exists. .. note:: @@ -1868,6 +1973,8 @@ features: This function handles UNC paths correctly. + .. audit-event:: os.mkdir path,mode,dir_fd os.makedirs + .. versionadded:: 3.2 The *exist_ok* parameter. @@ -1996,6 +2103,10 @@ features: This function can also support :ref:`paths relative to directory descriptors `. + When trying to resolve a path that may contain links, use + :func:`~os.path.realpath` to properly handle recursion and platform + differences. + .. availability:: Unix, Windows. .. versionchanged:: 3.2 @@ -2010,10 +2121,15 @@ features: .. versionchanged:: 3.8 Accepts a :term:`path-like object` and a bytes object on Windows. + .. versionchanged:: 3.8 + Added support for directory junctions, and changed to return the + substitution path (which typically includes ``\\?\`` prefix) rather + than the optional "print name" field that was previously returned. + .. function:: remove(path, *, dir_fd=None) - Remove (delete) the file *path*. If *path* is a directory, :exc:`OSError` is - raised. Use :func:`rmdir` to remove directories. + Remove (delete) the file *path*. If *path* is a directory, an + :exc:`IsADirectoryError` is raised. Use :func:`rmdir` to remove directories. This function can support :ref:`paths relative to directory descriptors `. @@ -2024,6 +2140,8 @@ features: This function is semantically identical to :func:`unlink`. + .. audit-event:: os.remove path,dir_fd os.remove + .. versionadded:: 3.3 The *dir_fd* argument. @@ -2044,25 +2162,35 @@ features: they are empty. Raises :exc:`OSError` if the leaf directory could not be successfully removed. + .. audit-event:: os.remove path,dir_fd os.removedirs + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. .. function:: rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None) - Rename the file or directory *src* to *dst*. If *dst* is a directory, - :exc:`OSError` will be raised. On Unix, if *dst* exists and is a file, it will - be replaced silently if the user has permission. The operation may fail on some - Unix flavors if *src* and *dst* are on different filesystems. If successful, - the renaming will be an atomic operation (this is a POSIX requirement). On - Windows, if *dst* already exists, :exc:`OSError` will be raised even if it is a - file. + Rename the file or directory *src* to *dst*. If *dst* exists, the operation + will fail with an :exc:`OSError` subclass in a number of cases: + + On Windows, if *dst* exists a :exc:`FileExistsError` is always raised. + + On Unix, if *src* is a file and *dst* is a directory or vice-versa, an + :exc:`IsADirectoryError` or a :exc:`NotADirectoryError` will be raised + respectively. If both are directories and *dst* is empty, *dst* will be + silently replaced. If *dst* is a non-empty directory, an :exc:`OSError` + is raised. If both are files, *dst* it will be replaced silently if the user + has permission. The operation may fail on some Unix flavors if *src* and + *dst* are on different filesystems. If successful, the renaming will be an + atomic operation (this is a POSIX requirement). This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to supply :ref:`paths relative to directory descriptors `. If you want cross-platform overwriting of the destination, use :func:`replace`. + .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.rename + .. versionadded:: 3.3 The *src_dir_fd* and *dst_dir_fd* arguments. @@ -2082,6 +2210,8 @@ features: This function can fail with the new directory structure made if you lack permissions needed to remove the leaf directory or file. + .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.renames + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *old* and *new*. @@ -2097,6 +2227,8 @@ features: This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to supply :ref:`paths relative to directory descriptors `. + .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.replace + .. versionadded:: 3.3 .. versionchanged:: 3.6 @@ -2105,13 +2237,16 @@ features: .. function:: rmdir(path, *, dir_fd=None) - Remove (delete) the directory *path*. Only works when the directory is - empty, otherwise, :exc:`OSError` is raised. In order to remove whole - directory trees, :func:`shutil.rmtree` can be used. + Remove (delete) the directory *path*. If the directory does not exist or is + not empty, an :exc:`FileNotFoundError` or an :exc:`OSError` is raised + respectively. In order to remove whole directory trees, + :func:`shutil.rmtree` can be used. This function can support :ref:`paths relative to directory descriptors `. + .. audit-event:: os.rmdir path,dir_fd os.rmdir + .. versionadded:: 3.3 The *dir_fd* parameter. @@ -2124,7 +2259,9 @@ features: Return an iterator of :class:`os.DirEntry` objects corresponding to the entries in the directory given by *path*. The entries are yielded in arbitrary order, and the special entries ``'.'`` and ``'..'`` are not - included. + included. If a file is removed from or added to the directory after + creating the iterator, whether an entry for that file be included is + unspecified. Using :func:`scandir` instead of :func:`listdir` can significantly increase the performance of code that also needs file type or file @@ -2145,6 +2282,8 @@ features: This function can also support :ref:`specifying a file descriptor `; the file descriptor must refer to a directory. + .. audit-event:: os.scandir path os.scandir + The :func:`scandir` iterator supports the :term:`context manager` protocol and has the following method: @@ -2314,7 +2453,8 @@ features: On Unix, this method always requires a system call. On Windows, it only requires a system call if *follow_symlinks* is ``True`` and the - entry is a symbolic link. + entry is a reparse point (for example, a symbolic link or directory + junction). On Windows, the ``st_ino``, ``st_dev`` and ``st_nlink`` attributes of the :class:`stat_result` are always set to zero. Call :func:`os.stat` to @@ -2351,6 +2491,17 @@ features: This function can support :ref:`specifying a file descriptor ` and :ref:`not following symlinks `. + On Windows, passing ``follow_symlinks=False`` will disable following all + name-surrogate reparse points, which includes symlinks and directory + junctions. Other types of reparse points that do not resemble links or that + the operating system is unable to follow will be opened directly. When + following a chain of multiple links, this may result in the original link + being returned instead of the non-link that prevented full traversal. To + obtain stat results for the final path in this case, use the + :func:`os.path.realpath` function to resolve the path name as far as + possible and call :func:`lstat` on the result. This does not apply to + dangling symlinks or junction points, which will raise the usual exceptions. + .. index:: module: stat Example:: @@ -2375,6 +2526,14 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.8 + On Windows, all reparse points that can be resolved by the operating + system are now followed, and passing ``follow_symlinks=False`` + disables following all name surrogate reparse points. If the operating + system reaches a reparse point that it is not able to follow, *stat* now + returns the information for the original path as if + ``follow_symlinks=False`` had been specified instead of raising an error. + .. class:: stat_result @@ -2526,7 +2685,7 @@ features: File type. - On Windows systems, the following attribute is also available: + On Windows systems, the following attributes are also available: .. attribute:: st_file_attributes @@ -2535,6 +2694,12 @@ features: :c:func:`GetFileInformationByHandle`. See the ``FILE_ATTRIBUTE_*`` constants in the :mod:`stat` module. + .. attribute:: st_reparse_tag + + When :attr:`st_file_attributes` has the ``FILE_ATTRIBUTE_REPARSE_POINT`` + set, this field contains the tag identifying the type of reparse point. + See the ``IO_REPARSE_TAG_*`` constants in the :mod:`stat` module. + The standard module :mod:`stat` defines functions and constants that are useful for extracting information from a :c:type:`stat` structure. (On Windows, some items are filled with dummy values.) @@ -2562,6 +2727,14 @@ features: .. versionadded:: 3.7 Added the :attr:`st_fstype` member to Solaris/derivatives. + .. versionadded:: 3.8 + Added the :attr:`st_reparse_tag` member on Windows. + + .. versionchanged:: 3.8 + On Windows, the :attr:`st_mode` member now identifies special + files as :const:`S_IFCHR`, :const:`S_IFIFO` or :const:`S_IFBLK` + as appropriate. + .. function:: statvfs(path) Perform a :c:func:`statvfs` system call on the given path. The return value is @@ -2593,7 +2766,7 @@ features: The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*. + Added support for specifying *path* as an open file descriptor. .. versionchanged:: 3.4 The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, @@ -2610,59 +2783,61 @@ features: .. data:: supports_dir_fd - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of their *dir_fd* parameter. Different platforms - provide different functionality, and an option that might work on one might - be unsupported on another. For consistency's sakes, functions that support - *dir_fd* always allow specifying the parameter, but will raise an exception - if the functionality is not actually available. - - To check whether a particular function permits use of its *dir_fd* - parameter, use the ``in`` operator on ``supports_dir_fd``. As an example, - this expression determines whether the *dir_fd* parameter of :func:`os.stat` - is locally available:: + A :class:`set` object indicating which functions in the :mod:`os` + module accept an open file descriptor for their *dir_fd* parameter. + Different platforms provide different features, and the underlying + functionality Python uses to implement the *dir_fd* parameter is not + available on all platforms Python supports. For consistency's sake, + functions that may support *dir_fd* always allow specifying the + parameter, but will throw an exception if the functionality is used + when it's not locally available. (Specifying ``None`` for *dir_fd* + is always supported on all platforms.) + + To check whether a particular function accepts an open file descriptor + for its *dir_fd* parameter, use the ``in`` operator on ``supports_dir_fd``. + As an example, this expression evaluates to ``True`` if :func:`os.stat` + accepts open file descriptors for *dir_fd* on the local platform:: os.stat in os.supports_dir_fd - Currently *dir_fd* parameters only work on Unix platforms; none of them work - on Windows. + Currently *dir_fd* parameters only work on Unix platforms; + none of them work on Windows. .. versionadded:: 3.3 .. data:: supports_effective_ids - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of the *effective_ids* parameter for - :func:`os.access`. If the local platform supports it, the collection will - contain :func:`os.access`, otherwise it will be empty. + A :class:`set` object indicating whether :func:`os.access` permits + specifying ``True`` for its *effective_ids* parameter on the local platform. + (Specifying ``False`` for *effective_ids* is always supported on all + platforms.) If the local platform supports it, the collection will contain + :func:`os.access`; otherwise it will be empty. - To check whether you can use the *effective_ids* parameter for - :func:`os.access`, use the ``in`` operator on ``supports_effective_ids``, - like so:: + This expression evaluates to ``True`` if :func:`os.access` supports + ``effective_ids=True`` on the local platform:: os.access in os.supports_effective_ids - Currently *effective_ids* only works on Unix platforms; it does not work on - Windows. + Currently *effective_ids* is only supported on Unix platforms; + it does not work on Windows. .. versionadded:: 3.3 .. data:: supports_fd - A :class:`~collections.abc.Set` object indicating which functions in the + A :class:`set` object indicating which functions in the :mod:`os` module permit specifying their *path* parameter as an open file - descriptor. Different platforms provide different functionality, and an - option that might work on one might be unsupported on another. For - consistency's sakes, functions that support *fd* always allow specifying - the parameter, but will raise an exception if the functionality is not - actually available. + descriptor on the local platform. Different platforms provide different + features, and the underlying functionality Python uses to accept open file + descriptors as *path* arguments is not available on all platforms Python + supports. - To check whether a particular function permits specifying an open file + To determine whether a particular function permits specifying an open file descriptor for its *path* parameter, use the ``in`` operator on - ``supports_fd``. As an example, this expression determines whether - :func:`os.chdir` accepts open file descriptors when called on your local + ``supports_fd``. As an example, this expression evaluates to ``True`` if + :func:`os.chdir` accepts open file descriptors for *path* on your local platform:: os.chdir in os.supports_fd @@ -2672,17 +2847,21 @@ features: .. data:: supports_follow_symlinks - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of their *follow_symlinks* parameter. Different - platforms provide different functionality, and an option that might work on - one might be unsupported on another. For consistency's sakes, functions that - support *follow_symlinks* always allow specifying the parameter, but will - raise an exception if the functionality is not actually available. - - To check whether a particular function permits use of its *follow_symlinks* - parameter, use the ``in`` operator on ``supports_follow_symlinks``. As an - example, this expression determines whether the *follow_symlinks* parameter - of :func:`os.stat` is locally available:: + A :class:`set` object indicating which functions in the :mod:`os` module + accept ``False`` for their *follow_symlinks* parameter on the local platform. + Different platforms provide different features, and the underlying + functionality Python uses to implement *follow_symlinks* is not available + on all platforms Python supports. For consistency's sake, functions that + may support *follow_symlinks* always allow specifying the parameter, but + will throw an exception if the functionality is used when it's not locally + available. (Specifying ``True`` for *follow_symlinks* is always supported + on all platforms.) + + To check whether a particular function accepts ``False`` for its + *follow_symlinks* parameter, use the ``in`` operator on + ``supports_follow_symlinks``. As an example, this expression evaluates + to ``True`` if you may specify ``follow_symlinks=False`` when calling + :func:`os.stat` on the local platform:: os.stat in os.supports_follow_symlinks @@ -2713,6 +2892,8 @@ features: :exc:`OSError` is raised when the function is called by an unprivileged user. + .. audit-event:: os.symlink src,dst,dir_fd os.symlink + .. availability:: Unix, Windows. .. versionchanged:: 3.2 @@ -2745,6 +2926,8 @@ features: This function can support :ref:`specifying a file descriptor `. + .. audit-event:: os.truncate path,length os.truncate + .. availability:: Unix, Windows. .. versionadded:: 3.3 @@ -2763,6 +2946,8 @@ features: traditional Unix name. Please see the documentation for :func:`remove` for further information. + .. audit-event:: os.remove path,dir_fd os.unlink + .. versionadded:: 3.3 The *dir_fd* parameter. @@ -2800,8 +2985,10 @@ features: :ref:`paths relative to directory descriptors ` and :ref:`not following symlinks `. + .. audit-event:: os.utime path,times,ns,dir_fd os.utime + .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*, + Added support for specifying *path* as an open file descriptor, and the *dir_fd*, *follow_symlinks*, and *ns* parameters. .. versionchanged:: 3.6 @@ -2824,7 +3011,10 @@ features: *filenames* is a list of the names of the non-directory files in *dirpath*. Note that the names in the lists contain no path components. To get a full path (which begins with *top*) to a file or directory in *dirpath*, do - ``os.path.join(dirpath, name)``. + ``os.path.join(dirpath, name)``. Whether or not the lists are sorted + depends on the file system. If a file is removed from or added to the + *dirpath* directory during generating the lists, whether a name for that + file be included is unspecified. If optional argument *topdown* is ``True`` or not specified, the triple for a directory is generated before the triples for any of its subdirectories @@ -2893,6 +3083,8 @@ features: for name in dirs: os.rmdir(os.path.join(root, name)) + .. audit-event:: os.walk top,topdown,onerror,followlinks os.walk + .. versionchanged:: 3.5 This function now calls :func:`os.scandir` instead of :func:`os.listdir`, making it faster by reducing the number of calls to :func:`os.stat`. @@ -2952,6 +3144,8 @@ features: for name in dirs: os.rmdir(name, dir_fd=rootfd) + .. audit-event:: os.fwalk top,topdown,onerror,follow_symlinks,dir_fd os.fwalk + .. availability:: Unix. .. versionadded:: 3.3 @@ -2963,6 +3157,51 @@ features: Added support for :class:`bytes` paths. +.. function:: memfd_create(name[, flags=os.MFD_CLOEXEC]) + + Create an anonymous file and return a file descriptor that refers to it. + *flags* must be one of the ``os.MFD_*`` constants available on the system + (or a bitwise ORed combination of them). By default, the new file + descriptor is :ref:`non-inheritable `. + + The name supplied in *name* is used as a filename and will be displayed as + the target of the corresponding symbolic link in the directory + ``/proc/self/fd/``. The displayed name is always prefixed with ``memfd:`` + and serves only for debugging purposes. Names do not affect the behavior of + the file descriptor, and as such multiple files can have the same name + without any side effects. + + .. availability:: Linux 3.17 or newer with glibc 2.27 or newer. + + .. versionadded:: 3.8 + + +.. data:: MFD_CLOEXEC + MFD_ALLOW_SEALING + MFD_HUGETLB + MFD_HUGE_SHIFT + MFD_HUGE_MASK + MFD_HUGE_64KB + MFD_HUGE_512KB + MFD_HUGE_1MB + MFD_HUGE_2MB + MFD_HUGE_8MB + MFD_HUGE_16MB + MFD_HUGE_32MB + MFD_HUGE_256MB + MFD_HUGE_512MB + MFD_HUGE_1GB + MFD_HUGE_2GB + MFD_HUGE_16GB + + These flags can be passed to :func:`memfd_create`. + + .. availability:: Linux 3.17 or newer with glibc 2.27 or newer. The + ``MFD_HUGE*`` flags are only available since Linux 4.14. + + .. versionadded:: 3.8 + + Linux extended attributes ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2980,6 +3219,8 @@ These functions are all available on Linux only. This function can support :ref:`specifying a file descriptor ` and :ref:`not following symlinks `. + .. audit-event:: os.getxattr path,attribute os.getxattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *path* and *attribute*. @@ -2994,6 +3235,8 @@ These functions are all available on Linux only. This function can support :ref:`specifying a file descriptor ` and :ref:`not following symlinks `. + .. audit-event:: os.listxattr path os.listxattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -3008,6 +3251,8 @@ These functions are all available on Linux only. This function can support :ref:`specifying a file descriptor ` and :ref:`not following symlinks `. + .. audit-event:: os.removexattr path,attribute os.removexattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *path* and *attribute*. @@ -3031,6 +3276,8 @@ These functions are all available on Linux only. A bug in Linux kernel versions less than 2.6.39 caused the flags argument to be ignored on some filesystems. + .. audit-event:: os.setxattr path,attribute,value,flags os.setxattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *path* and *attribute*. @@ -3093,6 +3340,8 @@ to be ignored. `_ for more information about how DLLs are loaded. + .. audit-event:: os.add_dll_directory path os.add_dll_directory + .. availability:: Windows. .. versionadded:: 3.8 @@ -3159,10 +3408,12 @@ to be ignored. you can check whether or not it is available using :data:`os.supports_fd`. If it is unavailable, using it will raise a :exc:`NotImplementedError`. + .. audit-event:: os.exec path,args,env os.execl + .. availability:: Unix, Windows. .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path* + Added support for specifying *path* as an open file descriptor for :func:`execve`. .. versionchanged:: 3.6 @@ -3321,7 +3572,13 @@ written in Python, such as a mail server's external command delivery program. parent. If an error occurs :exc:`OSError` is raised. Note that some platforms including FreeBSD <= 6.3 and Cygwin have - known issues when using fork() from a thread. + known issues when using ``fork()`` from a thread. + + .. audit-event:: os.fork "" os.fork + + .. versionchanged:: 3.8 + Calling ``fork()`` in a subinterpreter is no longer supported + (:exc:`RuntimeError` is raised). .. warning:: @@ -3338,6 +3595,12 @@ written in Python, such as a mail server's external command delivery program. master end of the pseudo-terminal. For a more portable approach, use the :mod:`pty` module. If an error occurs :exc:`OSError` is raised. + .. audit-event:: os.forkpty "" os.forkpty + + .. versionchanged:: 3.8 + Calling ``forkpty()`` in a subinterpreter is no longer supported + (:exc:`RuntimeError` is raised). + .. availability:: some flavors of Unix. @@ -3360,6 +3623,8 @@ written in Python, such as a mail server's external command delivery program. See also :func:`signal.pthread_kill`. + .. audit-event:: os.kill pid,sig os.kill + .. versionadded:: 3.2 Windows support. @@ -3372,6 +3637,8 @@ written in Python, such as a mail server's external command delivery program. Send the signal *sig* to the process group *pgid*. + .. audit-event:: os.killpg pgid,sig os.killpg + .. availability:: Unix. @@ -3382,6 +3649,19 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Unix. +.. function:: pidfd_open(pid, flags=0) + + Return a file descriptor referring to the process *pid*. This descriptor can + be used to perform process management without races and signals. The *flags* + argument is provided for future extensions; no flag values are currently + defined. + + See the :manpage:`pidfd_open(2)` man page for more details. + + .. availability:: Linux 5.3+ + .. versionadded:: 3.9 + + .. function:: plock(op) Lock program segments into memory. The value of *op* (defined in @@ -3409,6 +3689,11 @@ written in Python, such as a mail server's external command delivery program. subprocess was killed.) On Windows systems, the return value contains the signed integer return code from the child process. + On Unix, :func:`waitstatus_to_exitcode` can be used to convert the ``close`` + method result (exit status) into an exit code if it is not ``None``. On + Windows, the ``close`` method result is directly the exit code + (or ``None``). + This is implemented using :class:`subprocess.Popen`; see that class's documentation for more powerful ways to manage and communicate with subprocesses. @@ -3494,6 +3779,8 @@ written in Python, such as a mail server's external command delivery program. :c:data:`POSIX_SPAWN_SETSCHEDPARAM` and :c:data:`POSIX_SPAWN_SETSCHEDULER` flags. + .. audit-event:: os.posix_spawn path,argv,env os.posix_spawn + .. versionadded:: 3.8 .. availability:: Unix. @@ -3508,6 +3795,8 @@ written in Python, such as a mail server's external command delivery program. for the *executable* file in the list of directories specified by the :envvar:`PATH` environment variable (in the same way as for ``execvp(3)``). + .. audit-event:: os.posix_spawn path,argv,env os.posix_spawnp + .. versionadded:: 3.8 .. availability:: See :func:`posix_spawn` documentation. @@ -3568,6 +3857,9 @@ written in Python, such as a mail server's external command delivery program. process. On Windows, the process id will actually be the process handle, so can be used with the :func:`waitpid` function. + Note on VxWorks, this function doesn't return ``-signal`` when the new process is + killed. Instead it raises OSError exception. + The "l" and "v" variants of the :func:`spawn\* ` functions differ in how command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the @@ -3605,6 +3897,8 @@ written in Python, such as a mail server's external command delivery program. L = ['cp', 'index.html', '/dev/null'] os.spawnvpe(os.P_WAIT, 'cp', L, os.environ) + .. audit-event:: os.spawn mode,path,args,env os.spawnl + .. availability:: Unix, Windows. :func:`spawnlp`, :func:`spawnlpe`, :func:`spawnvp` and :func:`spawnvpe` are not available on Windows. :func:`spawnle` and :func:`spawnve` are not thread-safe on Windows; we advise you to use the @@ -3674,6 +3968,8 @@ written in Python, such as a mail server's external command delivery program. function is not resolved until this function is first called. If the function cannot be resolved, :exc:`NotImplementedError` will be raised. + .. audit-event:: os.startfile path,operation os.startfile + .. availability:: Windows. @@ -3701,6 +3997,12 @@ written in Python, such as a mail server's external command delivery program. to using this function. See the :ref:`subprocess-replacements` section in the :mod:`subprocess` documentation for some helpful recipes. + On Unix, :func:`waitstatus_to_exitcode` can be used to convert the result + (exit status) into an exit code. On Windows, the result is directly the exit + code. + + .. audit-event:: os.system command os.system + .. availability:: Unix, Windows. @@ -3720,9 +4022,9 @@ written in Python, such as a mail server's external command delivery program. :attr:`children_system`, and :attr:`elapsed` in that order. See the Unix manual page - :manpage:`times(2)` or the corresponding Windows Platform API documentation. - On Windows, only :attr:`user` and :attr:`system` are known; the other - attributes are zero. + :manpage:`times(2)` and :manpage:`times(3)` manual page on Unix or `the GetProcessTimes MSDN + `_ + on Windows. On Windows, only :attr:`user` and :attr:`system` are known; the other attributes are zero. .. availability:: Unix, Windows. @@ -3739,12 +4041,21 @@ written in Python, such as a mail server's external command delivery program. number is zero); the high bit of the low byte is set if a core file was produced. + :func:`waitstatus_to_exitcode` can be used to convert the exit status into an + exit code. + .. availability:: Unix. + .. seealso:: + + :func:`waitpid` can be used to wait for the completion of a specific + child process and has more options. + .. function:: waitid(idtype, id, options) Wait for the completion of one or more child processes. - *idtype* can be :data:`P_PID`, :data:`P_PGID` or :data:`P_ALL`. + *idtype* can be :data:`P_PID`, :data:`P_PGID`, :data:`P_ALL`, or + :data:`P_PIDFD` on Linux. *id* specifies the pid to wait on. *options* is constructed from the ORing of one or more of :data:`WEXITED`, :data:`WSTOPPED` or :data:`WCONTINUED` and additionally may be ORed with @@ -3769,6 +4080,15 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.3 +.. data:: P_PIDFD + + This is a Linux-specific *idtype* that indicates that *id* is a file + descriptor that refers to a process. + + .. availability:: Linux 5.4+ + + .. versionadded:: 3.9 + .. data:: WEXITED WSTOPPED WNOWAIT @@ -3782,8 +4102,10 @@ written in Python, such as a mail server's external command delivery program. .. data:: CLD_EXITED + CLD_KILLED CLD_DUMPED CLD_TRAPPED + CLD_STOPPED CLD_CONTINUED These are the possible values for :attr:`si_code` in the result returned by @@ -3793,6 +4115,9 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.3 + .. versionchanged:: 3.9 + Added :data:`CLD_KILLED` and :data:`CLD_STOPPED` values. + .. function:: waitpid(pid, options) @@ -3821,6 +4146,9 @@ written in Python, such as a mail server's external command delivery program. id is known, not necessarily a child process. The :func:`spawn\* ` functions called with :const:`P_NOWAIT` return suitable process handles. + :func:`waitstatus_to_exitcode` can be used to convert the exit status into an + exit code. + .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an exception, the function now retries the system call instead of raising an @@ -3836,6 +4164,9 @@ written in Python, such as a mail server's external command delivery program. information. The option argument is the same as that provided to :func:`waitpid` and :func:`wait4`. + :func:`waitstatus_to_exitcode` can be used to convert the exit status into an + exitcode. + .. availability:: Unix. @@ -3847,9 +4178,42 @@ written in Python, such as a mail server's external command delivery program. resource usage information. The arguments to :func:`wait4` are the same as those provided to :func:`waitpid`. + :func:`waitstatus_to_exitcode` can be used to convert the exit status into an + exitcode. + .. availability:: Unix. +.. function:: waitstatus_to_exitcode(status) + + Convert a wait status to an exit code. + + On Unix: + + * If the process exited normally (if ``WIFEXITED(status)`` is true), + return the process exit status (return ``WEXITSTATUS(status)``): + result greater than or equal to 0. + * If the process was terminated by a signal (if ``WIFSIGNALED(status)`` is + true), return ``-signum`` where *signum* is the number of the signal that + caused the process to terminate (return ``-WTERMSIG(status)``): + result less than 0. + * Otherwise, raise a :exc:`ValueError`. + + On Windows, return *status* shifted right by 8 bits. + + On Unix, if the process is being traced or if :func:`waitpid` was called + with :data:`WUNTRACED` option, the caller must first check if + ``WIFSTOPPED(status)`` is true. This function must not be called if + ``WIFSTOPPED(status)`` is true. + + .. seealso:: + + :func:`WIFEXITED`, :func:`WEXITSTATUS`, :func:`WIFSIGNALED`, + :func:`WTERMSIG`, :func:`WIFSTOPPED`, :func:`WSTOPSIG` functions. + + .. versionadded:: 3.9 + + .. data:: WNOHANG The option for :func:`waitpid` to return immediately if no child process status @@ -3883,28 +4247,36 @@ used to determine the disposition of a process. Return ``True`` if a core dump was generated for the process, otherwise return ``False``. + This function should be employed only if :func:`WIFSIGNALED` is true. + .. availability:: Unix. .. function:: WIFCONTINUED(status) - Return ``True`` if the process has been continued from a job control stop, - otherwise return ``False``. + Return ``True`` if a stopped child has been resumed by delivery of + :data:`~signal.SIGCONT` (if the process has been continued from a job + control stop), otherwise return ``False``. + + See :data:`WCONTINUED` option. .. availability:: Unix. .. function:: WIFSTOPPED(status) - Return ``True`` if the process has been stopped, otherwise return - ``False``. + Return ``True`` if the process was stopped by delivery of a signal, + otherwise return ``False``. - .. availability:: Unix. + :func:`WIFSTOPPED` only returns ``True`` if the :func:`waitpid` call was + done using :data:`WUNTRACED` option or when the process is being traced (see + :manpage:`ptrace(2)`). + .. availability:: Unix. .. function:: WIFSIGNALED(status) - Return ``True`` if the process exited due to a signal, otherwise return + Return ``True`` if the process was terminated by a signal, otherwise return ``False``. .. availability:: Unix. @@ -3912,7 +4284,8 @@ used to determine the disposition of a process. .. function:: WIFEXITED(status) - Return ``True`` if the process exited using the :manpage:`exit(2)` system call, + Return ``True`` if the process exited terminated normally, that is, + by calling ``exit()`` or ``_exit()``, or by returning from ``main()``; otherwise return ``False``. .. availability:: Unix. @@ -3920,8 +4293,9 @@ used to determine the disposition of a process. .. function:: WEXITSTATUS(status) - If ``WIFEXITED(status)`` is true, return the integer parameter to the - :manpage:`exit(2)` system call. Otherwise, the return value is meaningless. + Return the process exit status. + + This function should be employed only if :func:`WIFEXITED` is true. .. availability:: Unix. @@ -3930,12 +4304,16 @@ used to determine the disposition of a process. Return the signal which caused the process to stop. + This function should be employed only if :func:`WIFSTOPPED` is true. + .. availability:: Unix. .. function:: WTERMSIG(status) - Return the signal which caused the process to exit. + Return the number of the signal that caused the process to terminate. + + This function should be employed only if :func:`WIFSIGNALED` is true. .. availability:: Unix. diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index 4548459f8e261d..48c1f2754111aa 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -30,10 +30,11 @@ available for Python: for generating bindings for C++ libraries as Python classes, and is specifically designed for Python. - `PySide `_ - PySide is a newer binding to the Qt toolkit, provided by Nokia. - Compared to PyQt, its licensing scheme is friendlier to non-open source - applications. + `PySide2 `_ + Also known as the Qt for Python project, PySide2 is a newer binding to the + Qt toolkit. It is provided by The Qt Company and aims to provide a + complete port of PySide to Qt 5. Compared to PyQt, its licensing scheme is + friendlier to non-open source applications. `wxPython `_ wxPython is a cross-platform GUI toolkit for Python that is built around @@ -47,7 +48,7 @@ available for Python: an XML-based resource format and more, including an ever growing library of user-contributed modules. -PyGTK, PyQt, and wxPython, all have a modern look and feel and more +PyGTK, PyQt, PySide2, and wxPython, all have a modern look and feel and more widgets than Tkinter. In addition, there are many other GUI toolkits for Python, both cross-platform, and platform-specific. See the `GUI Programming `_ page in the Python Wiki for a diff --git a/Doc/library/parser.rst b/Doc/library/parser.rst deleted file mode 100644 index a302681eca0568..00000000000000 --- a/Doc/library/parser.rst +++ /dev/null @@ -1,356 +0,0 @@ -:mod:`parser` --- Access Python parse trees -=========================================== - -.. module:: parser - :synopsis: Access parse trees for Python source code. - -.. moduleauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Fred L. Drake, Jr. - -.. Copyright 1995 Virginia Polytechnic Institute and State University and Fred - L. Drake, Jr. This copyright notice must be distributed on all copies, but - this document otherwise may be distributed as part of the Python - distribution. No fee may be charged for this document in any representation, - either on paper or electronically. This restriction does not affect other - elements in a distributed package in any way. - -.. index:: single: parsing; Python source code - --------------- - -The :mod:`parser` module provides an interface to Python's internal parser and -byte-code compiler. The primary purpose for this interface is to allow Python -code to edit the parse tree of a Python expression and create executable code -from this. This is better than trying to parse and modify an arbitrary Python -code fragment as a string because parsing is performed in a manner identical to -the code forming the application. It is also faster. - -.. note:: - - From Python 2.5 onward, it's much more convenient to cut in at the Abstract - Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` - module. - -There are a few things to note about this module which are important to making -use of the data structures created. This is not a tutorial on editing the parse -trees for Python code, but some examples of using the :mod:`parser` module are -presented. - -Most importantly, a good understanding of the Python grammar processed by the -internal parser is required. For full information on the language syntax, refer -to :ref:`reference-index`. The parser -itself is created from a grammar specification defined in the file -:file:`Grammar/Grammar` in the standard Python distribution. The parse trees -stored in the ST objects created by this module are the actual output from the -internal parser when created by the :func:`expr` or :func:`suite` functions, -described below. The ST objects created by :func:`sequence2st` faithfully -simulate those structures. Be aware that the values of the sequences which are -considered "correct" will vary from one version of Python to another as the -formal grammar for the language is revised. However, transporting code from one -Python version to another as source text will always allow correct parse trees -to be created in the target version, with the only restriction being that -migrating to an older version of the interpreter will not support more recent -language constructs. The parse trees are not typically compatible from one -version to another, whereas source code has always been forward-compatible. - -Each element of the sequences returned by :func:`st2list` or :func:`st2tuple` -has a simple form. Sequences representing non-terminal elements in the grammar -always have a length greater than one. The first element is an integer which -identifies a production in the grammar. These integers are given symbolic names -in the C header file :file:`Include/graminit.h` and the Python module -:mod:`symbol`. Each additional element of the sequence represents a component -of the production as recognized in the input string: these are always sequences -which have the same form as the parent. An important aspect of this structure -which should be noted is that keywords used to identify the parent node type, -such as the keyword :keyword:`if` in an :const:`if_stmt`, are included in the -node tree without any special treatment. For example, the :keyword:`!if` keyword -is represented by the tuple ``(1, 'if')``, where ``1`` is the numeric value -associated with all :const:`NAME` tokens, including variable and function names -defined by the user. In an alternate form returned when line number information -is requested, the same token might be represented as ``(1, 'if', 12)``, where -the ``12`` represents the line number at which the terminal symbol was found. - -Terminal elements are represented in much the same way, but without any child -elements and the addition of the source text which was identified. The example -of the :keyword:`if` keyword above is representative. The various types of -terminal symbols are defined in the C header file :file:`Include/token.h` and -the Python module :mod:`token`. - -The ST objects are not required to support the functionality of this module, -but are provided for three purposes: to allow an application to amortize the -cost of processing complex parse trees, to provide a parse tree representation -which conserves memory space when compared to the Python list or tuple -representation, and to ease the creation of additional modules in C which -manipulate parse trees. A simple "wrapper" class may be created in Python to -hide the use of ST objects. - -The :mod:`parser` module defines functions for a few distinct purposes. The -most important purposes are to create ST objects and to convert ST objects to -other representations such as parse trees and compiled code objects, but there -are also functions which serve to query the type of parse tree represented by an -ST object. - - -.. seealso:: - - Module :mod:`symbol` - Useful constants representing internal nodes of the parse tree. - - Module :mod:`token` - Useful constants representing leaf nodes of the parse tree and functions for - testing node values. - - -.. _creating-sts: - -Creating ST Objects -------------------- - -ST objects may be created from source code or from a parse tree. When creating -an ST object from source, different functions are used to create the ``'eval'`` -and ``'exec'`` forms. - - -.. function:: expr(source) - - The :func:`expr` function parses the parameter *source* as if it were an input - to ``compile(source, 'file.py', 'eval')``. If the parse succeeds, an ST object - is created to hold the internal parse tree representation, otherwise an - appropriate exception is raised. - - -.. function:: suite(source) - - The :func:`suite` function parses the parameter *source* as if it were an input - to ``compile(source, 'file.py', 'exec')``. If the parse succeeds, an ST object - is created to hold the internal parse tree representation, otherwise an - appropriate exception is raised. - - -.. function:: sequence2st(sequence) - - This function accepts a parse tree represented as a sequence and builds an - internal representation if possible. If it can validate that the tree conforms - to the Python grammar and all nodes are valid node types in the host version of - Python, an ST object is created from the internal representation and returned - to the called. If there is a problem creating the internal representation, or - if the tree cannot be validated, a :exc:`ParserError` exception is raised. An - ST object created this way should not be assumed to compile correctly; normal - exceptions raised by compilation may still be initiated when the ST object is - passed to :func:`compilest`. This may indicate problems not related to syntax - (such as a :exc:`MemoryError` exception), but may also be due to constructs such - as the result of parsing ``del f(0)``, which escapes the Python parser but is - checked by the bytecode compiler. - - Sequences representing terminal tokens may be represented as either two-element - lists of the form ``(1, 'name')`` or as three-element lists of the form ``(1, - 'name', 56)``. If the third element is present, it is assumed to be a valid - line number. The line number may be specified for any subset of the terminal - symbols in the input tree. - - -.. function:: tuple2st(sequence) - - This is the same function as :func:`sequence2st`. This entry point is - maintained for backward compatibility. - - -.. _converting-sts: - -Converting ST Objects ---------------------- - -ST objects, regardless of the input used to create them, may be converted to -parse trees represented as list- or tuple- trees, or may be compiled into -executable code objects. Parse trees may be extracted with or without line -numbering information. - - -.. function:: st2list(st, line_info=False, col_info=False) - - This function accepts an ST object from the caller in *st* and returns a - Python list representing the equivalent parse tree. The resulting list - representation can be used for inspection or the creation of a new parse tree in - list form. This function does not fail so long as memory is available to build - the list representation. If the parse tree will only be used for inspection, - :func:`st2tuple` should be used instead to reduce memory consumption and - fragmentation. When the list representation is required, this function is - significantly faster than retrieving a tuple representation and converting that - to nested lists. - - If *line_info* is true, line number information will be included for all - terminal tokens as a third element of the list representing the token. Note - that the line number provided specifies the line on which the token *ends*. - This information is omitted if the flag is false or omitted. - - -.. function:: st2tuple(st, line_info=False, col_info=False) - - This function accepts an ST object from the caller in *st* and returns a - Python tuple representing the equivalent parse tree. Other than returning a - tuple instead of a list, this function is identical to :func:`st2list`. - - If *line_info* is true, line number information will be included for all - terminal tokens as a third element of the list representing the token. This - information is omitted if the flag is false or omitted. - - -.. function:: compilest(st, filename='') - - .. index:: - builtin: exec - builtin: eval - - The Python byte compiler can be invoked on an ST object to produce code objects - which can be used as part of a call to the built-in :func:`exec` or :func:`eval` - functions. This function provides the interface to the compiler, passing the - internal parse tree from *st* to the parser, using the source file name - specified by the *filename* parameter. The default value supplied for *filename* - indicates that the source was an ST object. - - Compiling an ST object may result in exceptions related to compilation; an - example would be a :exc:`SyntaxError` caused by the parse tree for ``del f(0)``: - this statement is considered legal within the formal grammar for Python but is - not a legal language construct. The :exc:`SyntaxError` raised for this - condition is actually generated by the Python byte-compiler normally, which is - why it can be raised at this point by the :mod:`parser` module. Most causes of - compilation failure can be diagnosed programmatically by inspection of the parse - tree. - - -.. _querying-sts: - -Queries on ST Objects ---------------------- - -Two functions are provided which allow an application to determine if an ST was -created as an expression or a suite. Neither of these functions can be used to -determine if an ST was created from source code via :func:`expr` or -:func:`suite` or from a parse tree via :func:`sequence2st`. - - -.. function:: isexpr(st) - - .. index:: builtin: compile - - When *st* represents an ``'eval'`` form, this function returns true, otherwise - it returns false. This is useful, since code objects normally cannot be queried - for this information using existing built-in functions. Note that the code - objects created by :func:`compilest` cannot be queried like this either, and - are identical to those created by the built-in :func:`compile` function. - - -.. function:: issuite(st) - - This function mirrors :func:`isexpr` in that it reports whether an ST object - represents an ``'exec'`` form, commonly known as a "suite." It is not safe to - assume that this function is equivalent to ``not isexpr(st)``, as additional - syntactic fragments may be supported in the future. - - -.. _st-errors: - -Exceptions and Error Handling ------------------------------ - -The parser module defines a single exception, but may also pass other built-in -exceptions from other portions of the Python runtime environment. See each -function for information about the exceptions it can raise. - - -.. exception:: ParserError - - Exception raised when a failure occurs within the parser module. This is - generally produced for validation failures rather than the built-in - :exc:`SyntaxError` raised during normal parsing. The exception argument is - either a string describing the reason of the failure or a tuple containing a - sequence causing the failure from a parse tree passed to :func:`sequence2st` - and an explanatory string. Calls to :func:`sequence2st` need to be able to - handle either type of exception, while calls to other functions in the module - will only need to be aware of the simple string values. - -Note that the functions :func:`compilest`, :func:`expr`, and :func:`suite` may -raise exceptions which are normally raised by the parsing and compilation -process. These include the built in exceptions :exc:`MemoryError`, -:exc:`OverflowError`, :exc:`SyntaxError`, and :exc:`SystemError`. In these -cases, these exceptions carry all the meaning normally associated with them. -Refer to the descriptions of each function for detailed information. - - -.. _st-objects: - -ST Objects ----------- - -Ordered and equality comparisons are supported between ST objects. Pickling of -ST objects (using the :mod:`pickle` module) is also supported. - - -.. data:: STType - - The type of the objects returned by :func:`expr`, :func:`suite` and - :func:`sequence2st`. - -ST objects have the following methods: - - -.. method:: ST.compile(filename='') - - Same as ``compilest(st, filename)``. - - -.. method:: ST.isexpr() - - Same as ``isexpr(st)``. - - -.. method:: ST.issuite() - - Same as ``issuite(st)``. - - -.. method:: ST.tolist(line_info=False, col_info=False) - - Same as ``st2list(st, line_info, col_info)``. - - -.. method:: ST.totuple(line_info=False, col_info=False) - - Same as ``st2tuple(st, line_info, col_info)``. - - -Example: Emulation of :func:`compile` -------------------------------------- - -While many useful operations may take place between parsing and bytecode -generation, the simplest operation is to do nothing. For this purpose, using -the :mod:`parser` module to produce an intermediate data structure is equivalent -to the code :: - - >>> code = compile('a + 5', 'file.py', 'eval') - >>> a = 5 - >>> eval(code) - 10 - -The equivalent operation using the :mod:`parser` module is somewhat longer, and -allows the intermediate internal parse tree to be retained as an ST object:: - - >>> import parser - >>> st = parser.expr('a + 5') - >>> code = st.compile('file.py') - >>> a = 5 - >>> eval(code) - 10 - -An application which needs both ST and code objects can package this code into -readily available functions:: - - import parser - - def load_suite(source_string): - st = parser.suite(source_string) - return st, st.compile() - - def load_expression(source_string): - st = parser.expr(source_string) - return st, st.compile() diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 450e8ff378a3a5..23486b625072f3 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -273,7 +273,7 @@ Methods and properties .. testsetup:: - from pathlib import PurePosixPath, PureWindowsPath + from pathlib import PurePath, PurePosixPath, PureWindowsPath Pure paths provide the following methods and properties: @@ -462,6 +462,19 @@ Pure paths provide the following methods and properties: True +.. method:: PurePath.is_relative_to(*other) + + Return whether or not this path is relative to the *other* path. + + >>> p = PurePath('/etc/passwd') + >>> p.is_relative_to('/etc') + True + >>> p.is_relative_to('/usr') + False + + .. versionadded:: 3.9 + + .. method:: PurePath.is_reserved() With :class:`PureWindowsPath`, return ``True`` if the path is considered @@ -515,8 +528,10 @@ Pure paths provide the following methods and properties: >>> PurePath('a/b.py').match('/*.py') False - As with other methods, case-sensitivity is observed:: + As with other methods, case-sensitivity follows platform defaults:: + >>> PurePosixPath('b.py').match('*.PY') + False >>> PureWindowsPath('b.py').match('*.PY') True @@ -536,7 +551,9 @@ Pure paths provide the following methods and properties: File "", line 1, in File "pathlib.py", line 694, in relative_to .format(str(self), str(formatted))) - ValueError: '/etc/passwd' does not start with '/usr' + ValueError: '/etc/passwd' is not in the subpath of '/usr' OR one path is relative and the other absolute. + + NOTE: This function is part of :class:`PurePath` and works with strings. It does not check or access the underlying file structure. .. method:: PurePath.with_name(name) @@ -556,6 +573,30 @@ Pure paths provide the following methods and properties: ValueError: PureWindowsPath('c:/') has an empty name +.. method:: PurePath.with_stem(stem) + + Return a new path with the :attr:`stem` changed. If the original path + doesn't have a name, ValueError is raised:: + + >>> p = PureWindowsPath('c:/Downloads/draft.txt') + >>> p.with_stem('final') + PureWindowsPath('c:/Downloads/final.txt') + >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') + >>> p.with_stem('lib') + PureWindowsPath('c:/Downloads/lib.gz') + >>> p = PureWindowsPath('c:/') + >>> p.with_stem('') + Traceback (most recent call last): + File "", line 1, in + File "/home/antoine/cpython/default/Lib/pathlib.py", line 861, in with_stem + return self.with_name(stem + self.suffix) + File "/home/antoine/cpython/default/Lib/pathlib.py", line 851, in with_name + raise ValueError("%r has an empty name" % (self,)) + ValueError: PureWindowsPath('c:/') has an empty name + + .. versionadded:: 3.9 + + .. method:: PurePath.with_suffix(suffix) Return a new path with the :attr:`suffix` changed. If the original path @@ -672,7 +713,7 @@ call fails (for example because the path doesn't exist). .. method:: Path.stat() - Return information about this path (similarly to :func:`os.stat`). + Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. The result is looked up at each call to this method. :: @@ -750,6 +791,8 @@ call fails (for example because the path doesn't exist). Using the "``**``" pattern in large directory trees may consume an inordinate amount of time. + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob + .. method:: Path.group() @@ -847,6 +890,11 @@ call fails (for example because the path doesn't exist). PosixPath('docs/_static') PosixPath('docs/Makefile') + The children are yielded in arbitrary order, and the special entries + ``'.'`` and ``'..'`` are not included. If a file is removed from or added + to the directory after creating the iterator, whether an path object for + that file be included is unspecified. + .. method:: Path.lchmod(mode) Like :meth:`Path.chmod` but, if the path points to a symbolic link, the @@ -931,25 +979,47 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 +.. method:: Path.readlink() + + Return the path to which the symbolic link points (as returned by + :func:`os.readlink`):: + + >>> p = Path('mylink') + >>> p.symlink_to('setup.py') + >>> p.readlink() + PosixPath('setup.py') + + .. versionadded:: 3.9 + + .. method:: Path.rename(target) - Rename this file or directory to the given *target*. On Unix, if - *target* exists and is a file, it will be replaced silently if the user - has permission. *target* can be either a string or another path object:: + Rename this file or directory to the given *target*, and return a new Path + instance pointing to *target*. On Unix, if *target* exists and is a file, + it will be replaced silently if the user has permission. *target* can be + either a string or another path object:: >>> p = Path('foo') >>> p.open('w').write('some text') 9 >>> target = Path('bar') >>> p.rename(target) + PosixPath('bar') >>> target.open().read() 'some text' + .. versionchanged:: 3.8 + Added return value, return the new Path instance. + .. method:: Path.replace(target) - Rename this file or directory to the given *target*. If *target* points - to an existing file or directory, it will be unconditionally replaced. + Rename this file or directory to the given *target*, and return a new Path + instance pointing to *target*. If *target* points to an existing file or + directory, it will be unconditionally replaced. + + .. versionchanged:: 3.8 + Added return value, return the new Path instance. .. method:: Path.resolve(strict=False) @@ -976,7 +1046,7 @@ call fails (for example because the path doesn't exist). is raised. .. versionadded:: 3.6 - The *strict* argument. + The *strict* argument (pre-3.6 behavior is strict). .. method:: Path.rglob(pattern) @@ -990,6 +1060,8 @@ call fails (for example because the path doesn't exist). PosixPath('setup.py'), PosixPath('test_pathlib.py')] + .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob + .. method:: Path.rmdir() @@ -1048,11 +1120,27 @@ call fails (for example because the path doesn't exist). otherwise :exc:`FileExistsError` is raised. -.. method:: Path.unlink() +.. method:: Path.unlink(missing_ok=False) Remove this file or symbolic link. If the path points to a directory, use :func:`Path.rmdir` instead. + If *missing_ok* is false (the default), :exc:`FileNotFoundError` is + raised if the path does not exist. + + If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be + ignored (same behavior as the POSIX ``rm -f`` command). + + .. versionchanged:: 3.8 + The *missing_ok* parameter was added. + + +.. method:: Path.link_to(target) + + Create a hard link pointing to a path named *target*. + + .. versionadded:: 3.8 + .. method:: Path.write_bytes(data) @@ -1081,6 +1169,9 @@ call fails (for example because the path doesn't exist). >>> p.read_text() 'Text file contents' + An existing file of the same name is overwritten. The optional parameters + have the same meaning as in :func:`open`. + .. versionadded:: 3.5 Correspondence to tools in the :mod:`os` module @@ -1101,6 +1192,7 @@ os and os.path pathlib :func:`os.path.abspath` :meth:`Path.resolve` :func:`os.chmod` :meth:`Path.chmod` :func:`os.mkdir` :meth:`Path.mkdir` +:func:`os.makedirs` :meth:`Path.mkdir` :func:`os.rename` :meth:`Path.rename` :func:`os.replace` :meth:`Path.replace` :func:`os.rmdir` :meth:`Path.rmdir` @@ -1109,9 +1201,13 @@ os and os.path pathlib :func:`os.path.exists` :meth:`Path.exists` :func:`os.path.expanduser` :meth:`Path.expanduser` and :meth:`Path.home` +:func:`os.listdir` :meth:`Path.iterdir` :func:`os.path.isdir` :meth:`Path.is_dir` :func:`os.path.isfile` :meth:`Path.is_file` :func:`os.path.islink` :meth:`Path.is_symlink` +:func:`os.link` :meth:`Path.link_to` +:func:`os.symlink` :meth:`Path.symlink_to` +:func:`os.readlink` :meth:`Path.readlink` :func:`os.stat` :meth:`Path.stat`, :meth:`Path.owner`, :meth:`Path.group` diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index c7864e9e3f2225..ed1e9712c0e3de 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -181,6 +181,8 @@ access further features, you have to do this yourself: import pdb; pdb.Pdb(skip=['django.*']).set_trace() + .. audit-event:: pdb.Pdb "" pdb.Pdb + .. versionadded:: 3.1 The *skip* argument. @@ -294,20 +296,20 @@ by the local file. Temporary breakpoint, which is removed automatically when it is first hit. The arguments are the same as for :pdbcmd:`break`. -.. pdbcommand:: cl(ear) [filename:lineno | bpnumber [bpnumber ...]] +.. pdbcommand:: cl(ear) [filename:lineno | bpnumber ...] With a *filename:lineno* argument, clear all the breakpoints at this line. With a space separated list of breakpoint numbers, clear those breakpoints. Without argument, clear all breaks (but first ask confirmation). -.. pdbcommand:: disable [bpnumber [bpnumber ...]] +.. pdbcommand:: disable [bpnumber ...] Disable the breakpoints given as a space separated list of breakpoint numbers. Disabling a breakpoint means it cannot cause the program to stop execution, but unlike clearing a breakpoint, it remains in the list of breakpoints and can be (re-)enabled. -.. pdbcommand:: enable [bpnumber [bpnumber ...]] +.. pdbcommand:: enable [bpnumber ...] Enable the breakpoints specified. @@ -529,6 +531,15 @@ by the local file. Quit from the debugger. The program being executed is aborted. +.. pdbcommand:: debug code + + Enter a recursive debugger that steps through the code + argument (which is an arbitrary expression or statement to be + executed in the current environment). + +.. pdbcommand:: retval + + Print the return value for the last return of a function. .. rubric:: Footnotes diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 53eb5d39ef9414..b7c34527719486 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -30,9 +30,17 @@ avoid confusion, the terms used here are "pickling" and "unpickling". .. warning:: - The :mod:`pickle` module is not secure against erroneous or maliciously - constructed data. Never unpickle data received from an untrusted or - unauthenticated source. + The ``pickle`` module **is not secure**. Only unpickle data you trust. + + It is possible to construct malicious pickle data which will **execute + arbitrary code during unpickling**. Never unpickle data that could have come + from an untrusted source, or that could have been tampered with. + + Consider signing data with :mod:`hmac` if you need to ensure that it has not + been tampered with. + + Safer serialization formats such as :mod:`json` may be more appropriate if + you are processing untrusted data. See :ref:`comparison-with-json`. Relationship to other Python modules @@ -75,6 +83,9 @@ The :mod:`pickle` module differs from :mod:`marshal` in several significant ways pickling and unpickling code deals with Python 2 to Python 3 type differences if your data is crossing that unique breaking change language boundary. + +.. _comparison-with-json: + Comparison with ``json`` ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -94,7 +105,10 @@ There are fundamental differences between the pickle protocols and types, and no custom classes; pickle can represent an extremely large number of Python types (many of them automatically, by clever usage of Python's introspection facilities; complex cases can be tackled by - implementing :ref:`specific object APIs `). + implementing :ref:`specific object APIs `); + +* Unlike pickle, deserializing untrusted JSON does not in itself create an + arbitrary code execution vulnerability. .. seealso:: The :mod:`json` module: a standard library module allowing JSON @@ -122,7 +136,7 @@ The module :mod:`pickletools` contains tools for analyzing data streams generated by :mod:`pickle`. :mod:`pickletools` source code has extensive comments about opcodes used by pickle protocols. -There are currently 5 different protocols which can be used for pickling. +There are currently 6 different protocols which can be used for pickling. The higher the protocol used, the more recent the version of Python needed to read the pickle produced. @@ -146,6 +160,10 @@ to read the pickle produced. Refer to :pep:`3154` for information about improvements brought by protocol 4. +* Protocol version 5 was added in Python 3.8. It adds support for out-of-band + data and speedup for in-band data. Refer to :pep:`574` for information about + improvements brought by protocol 5. + .. note:: Serialization is a more primitive notion than persistence; although :mod:`pickle` reads and writes file objects, it does not handle the issue of @@ -195,79 +213,59 @@ The :mod:`pickle` module provides the following constants: The :mod:`pickle` module provides the following functions to make the pickling process more convenient: -.. function:: dump(obj, file, protocol=None, \*, fix_imports=True) +.. function:: dump(obj, file, protocol=None, \*, fix_imports=True, buffer_callback=None) - Write a pickled representation of *obj* to the open :term:`file object` *file*. - This is equivalent to ``Pickler(file, protocol).dump(obj)``. + Write the pickled representation of the object *obj* to the open + :term:`file object` *file*. This is equivalent to + ``Pickler(file, protocol).dump(obj)``. - The optional *protocol* argument, an integer, tells the pickler to use - the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`. - If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative - number is specified, :data:`HIGHEST_PROTOCOL` is selected. + Arguments *file*, *protocol*, *fix_imports* and *buffer_callback* have + the same meaning as in the :class:`Pickler` constructor. - The *file* argument must have a write() method that accepts a single bytes - argument. It can thus be an on-disk file opened for binary writing, an - :class:`io.BytesIO` instance, or any other custom object that meets this - interface. - - If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3 names to the old module names used in Python 2, so - that the pickle data stream is readable with Python 2. + .. versionchanged:: 3.8 + The *buffer_callback* argument was added. -.. function:: dumps(obj, protocol=None, \*, fix_imports=True) +.. function:: dumps(obj, protocol=None, \*, fix_imports=True, buffer_callback=None) - Return the pickled representation of the object as a :class:`bytes` object, + Return the pickled representation of the object *obj* as a :class:`bytes` object, instead of writing it to a file. - Arguments *protocol* and *fix_imports* have the same meaning as in - :func:`dump`. + Arguments *protocol*, *fix_imports* and *buffer_callback* have the same + meaning as in the :class:`Pickler` constructor. -.. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict") + .. versionchanged:: 3.8 + The *buffer_callback* argument was added. - Read a pickled object representation from the open :term:`file object` +.. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) + + Read the pickled representation of an object from the open :term:`file object` *file* and return the reconstituted object hierarchy specified therein. This is equivalent to ``Unpickler(file).load()``. The protocol version of the pickle is detected automatically, so no - protocol argument is needed. Bytes past the pickled object's - representation are ignored. - - The argument *file* must have two methods, a read() method that takes an - integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file opened for - binary reading, an :class:`io.BytesIO` object, or any other custom object - that meets this interface. - - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. - Using ``encoding='latin1'`` is required for unpickling NumPy arrays and - instances of :class:`~datetime.datetime`, :class:`~datetime.date` and - :class:`~datetime.time` pickled by Python 2. + protocol argument is needed. Bytes past the pickled representation + of the object are ignored. + + Arguments *file*, *fix_imports*, *encoding*, *errors*, *strict* and *buffers* + have the same meaning as in the :class:`Unpickler` constructor. + + .. versionchanged:: 3.8 + The *buffers* argument was added. -.. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict") +.. function:: loads(data, /, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) - Read a pickled object hierarchy from a :class:`bytes` object and return the - reconstituted object hierarchy specified therein. + Return the reconstituted object hierarchy of the pickled representation + *data* of an object. *data* must be a :term:`bytes-like object`. The protocol version of the pickle is detected automatically, so no - protocol argument is needed. Bytes past the pickled object's - representation are ignored. - - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. - Using ``encoding='latin1'`` is required for unpickling NumPy arrays and - instances of :class:`~datetime.datetime`, :class:`~datetime.date` and - :class:`~datetime.time` pickled by Python 2. + protocol argument is needed. Bytes past the pickled representation + of the object are ignored. + + Arguments *file*, *fix_imports*, *encoding*, *errors*, *strict* and *buffers* + have the same meaning as in the :class:`Unpickler` constructor. + + .. versionchanged:: 3.8 + The *buffers* argument was added. The :mod:`pickle` module defines three exceptions: @@ -295,10 +293,10 @@ The :mod:`pickle` module defines three exceptions: IndexError. -The :mod:`pickle` module exports two classes, :class:`Pickler` and -:class:`Unpickler`: +The :mod:`pickle` module exports three classes, :class:`Pickler`, +:class:`Unpickler` and :class:`PickleBuffer`: -.. class:: Pickler(file, protocol=None, \*, fix_imports=True) +.. class:: Pickler(file, protocol=None, \*, fix_imports=True, buffer_callback=None) This takes a binary file for writing a pickle data stream. @@ -316,9 +314,23 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and map the new Python 3 names to the old module names used in Python 2, so that the pickle data stream is readable with Python 2. + If *buffer_callback* is None (the default), buffer views are + serialized into *file* as part of the pickle stream. + + If *buffer_callback* is not None, then it can be called any number + of times with a buffer view. If the callback returns a false value + (such as None), the given buffer is :ref:`out-of-band `; + otherwise the buffer is serialized in-band, i.e. inside the pickle stream. + + It is an error if *buffer_callback* is not None and *protocol* is + None or smaller than 5. + + .. versionchanged:: 3.8 + The *buffer_callback* argument was added. + .. method:: dump(obj) - Write a pickled representation of *obj* to the open file object given in + Write the pickled representation of *obj* to the open file object given in the constructor. .. method:: persistent_id(obj) @@ -356,6 +368,18 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and .. versionadded:: 3.3 + .. method:: reducer_override(self, obj) + + Special reducer that can be defined in :class:`Pickler` subclasses. This + method has priority over any reducer in the :attr:`dispatch_table`. It + should conform to the same interface as a :meth:`__reduce__` method, and + can optionally return ``NotImplemented`` to fallback on + :attr:`dispatch_table`-registered reducers to pickle ``obj``. + + For a detailed example, see :ref:`reducer_override`. + + .. versionadded:: 3.8 + .. attribute:: fast Deprecated. Enable fast mode if set to a true value. The fast mode @@ -367,32 +391,50 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and Use :func:`pickletools.optimize` if you need more compact pickles. -.. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict") +.. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) This takes a binary file for reading a pickle data stream. The protocol version of the pickle is detected automatically, so no protocol argument is needed. - The argument *file* must have two methods, a read() method that takes an - integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file object + The argument *file* must have three methods, a read() method that takes an + integer argument, a readinto() method that takes a buffer argument + and a readline() method that requires no arguments, as in the + :class:`io.BufferedIOBase` interface. Thus *file* can be an on-disk file opened for binary reading, an :class:`io.BytesIO` object, or any other custom object that meets this interface. - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can + The optional arguments *fix_imports*, *encoding* and *errors* are used + to control compatibility support for pickle stream generated by Python 2. + If *fix_imports* is true, pickle will try to map the old Python 2 names + to the new names used in Python 3. The *encoding* and *errors* tell + pickle how to decode 8-bit string instances pickled by Python 2; + these default to 'ASCII' and 'strict', respectively. The *encoding* can be 'bytes' to read these 8-bit string instances as bytes objects. + Using ``encoding='latin1'`` is required for unpickling NumPy arrays and + instances of :class:`~datetime.datetime`, :class:`~datetime.date` and + :class:`~datetime.time` pickled by Python 2. + + If *buffers* is None (the default), then all data necessary for + deserialization must be contained in the pickle stream. This means + that the *buffer_callback* argument was None when a :class:`Pickler` + was instantiated (or when :func:`dump` or :func:`dumps` was called). + + If *buffers* is not None, it should be an iterable of buffer-enabled + objects that is consumed each time the pickle stream references + an :ref:`out-of-band ` buffer view. Such buffers have been + given in order to the *buffer_callback* of a Pickler object. + + .. versionchanged:: 3.8 + The *buffers* argument was added. .. method:: load() - Read a pickled object representation from the open file object given in - the constructor, and return the reconstituted object hierarchy specified - therein. Bytes past the pickled object's representation are ignored. + Read the pickled representation of an object from the open file object + given in the constructor, and return the reconstituted object hierarchy + specified therein. Bytes past the pickled representation of the object + are ignored. .. method:: persistent_load(pid) @@ -415,6 +457,35 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and how they can be loaded, potentially reducing security risks. Refer to :ref:`pickle-restrict` for details. + .. audit-event:: pickle.find_class module,name pickle.Unpickler.find_class + +.. class:: PickleBuffer(buffer) + + A wrapper for a buffer representing picklable data. *buffer* must be a + :ref:`buffer-providing ` object, such as a + :term:`bytes-like object` or a N-dimensional array. + + :class:`PickleBuffer` is itself a buffer provider, therefore it is + possible to pass it to other APIs expecting a buffer-providing object, + such as :class:`memoryview`. + + :class:`PickleBuffer` objects can only be serialized using pickle + protocol 5 or higher. They are eligible for + :ref:`out-of-band serialization `. + + .. versionadded:: 3.8 + + .. method:: raw() + + Return a :class:`memoryview` of the memory area underlying this buffer. + The returned object is a one-dimensional, C-contiguous memoryview + with format ``B`` (unsigned bytes). :exc:`BufferError` is raised if + the buffer is neither C- nor Fortran-contiguous. + + .. method:: release() + + Release the underlying buffer exposed by the PickleBuffer object. + .. _pickle-picklable: @@ -568,9 +639,9 @@ the methods :meth:`__getstate__` and :meth:`__setstate__`. At unpickling time, some methods like :meth:`__getattr__`, :meth:`__getattribute__`, or :meth:`__setattr__` may be called upon the instance. In case those methods rely on some internal invariant being - true, the type should implement :meth:`__getnewargs__` or - :meth:`__getnewargs_ex__` to establish such an invariant; otherwise, - neither :meth:`__new__` nor :meth:`__init__` will be called. + true, the type should implement :meth:`__new__` to establish such an + invariant, as :meth:`__init__` is not called when unpickling an + instance. .. index:: pair: copy; protocol @@ -598,7 +669,7 @@ or both. module; the pickle module searches the module namespace to determine the object's module. This behaviour is typically useful for singletons. - When a tuple is returned, it must be between two and five items long. + When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their value. The semantics of each item are in order: @@ -629,6 +700,15 @@ or both. value``. This is primarily used for dictionary subclasses, but may be used by other classes as long as they implement :meth:`__setitem__`. + * Optionally, a callable with a ``(obj, state)`` signature. This + callable allows the user to programmatically control the state-updating + behavior of a specific object, instead of using ``obj``'s static + :meth:`__setstate__` method. If not ``None``, this callable will have + priority over ``obj``'s :meth:`__setstate__`. + + .. versionadded:: 3.8 + The optional sixth tuple item, ``(obj, state)``, was added. + .. method:: object.__reduce_ex__(protocol) @@ -657,13 +737,13 @@ alphanumeric characters (for protocol 0) [#]_ or just an arbitrary object (for any newer protocol). The resolution of such persistent IDs is not defined by the :mod:`pickle` -module; it will delegate this resolution to the user defined methods on the +module; it will delegate this resolution to the user-defined methods on the pickler and unpickler, :meth:`~Pickler.persistent_id` and :meth:`~Unpickler.persistent_load` respectively. -To pickle objects that have an external persistent id, the pickler must have a +To pickle objects that have an external persistent ID, the pickler must have a custom :meth:`~Pickler.persistent_id` method that takes an object as an -argument and returns either ``None`` or the persistent id for that object. +argument and returns either ``None`` or the persistent ID for that object. When ``None`` is returned, the pickler simply pickles the object as normal. When a persistent ID string is returned, the pickler will pickle that object, along with a marker so that the unpickler will recognize it as a persistent ID. @@ -782,6 +862,184 @@ A sample usage might be something like this:: >>> new_reader.readline() '3: Goodbye!' +.. _reducer_override: + +Custom Reduction for Types, Functions, and Other Objects +-------------------------------------------------------- + +.. versionadded:: 3.8 + +Sometimes, :attr:`~Pickler.dispatch_table` may not be flexible enough. +In particular we may want to customize pickling based on another criterion +than the object's type, or we may want to customize the pickling of +functions and classes. + +For those cases, it is possible to subclass from the :class:`Pickler` class and +implement a :meth:`~Pickler.reducer_override` method. This method can return an +arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return +``NotImplemented`` to fallback to the traditional behavior. + +If both the :attr:`~Pickler.dispatch_table` and +:meth:`~Pickler.reducer_override` are defined, then +:meth:`~Pickler.reducer_override` method takes priority. + +.. Note:: + For performance reasons, :meth:`~Pickler.reducer_override` may not be + called for the following objects: ``None``, ``True``, ``False``, and + exact instances of :class:`int`, :class:`float`, :class:`bytes`, + :class:`str`, :class:`dict`, :class:`set`, :class:`frozenset`, :class:`list` + and :class:`tuple`. + +Here is a simple example where we allow pickling and reconstructing +a given class:: + + import io + import pickle + + class MyClass: + my_attribute = 1 + + class MyPickler(pickle.Pickler): + def reducer_override(self, obj): + """Custom reducer for MyClass.""" + if getattr(obj, "__name__", None) == "MyClass": + return type, (obj.__name__, obj.__bases__, + {'my_attribute': obj.my_attribute}) + else: + # For any other object, fallback to usual reduction + return NotImplemented + + f = io.BytesIO() + p = MyPickler(f) + p.dump(MyClass) + + del MyClass + + unpickled_class = pickle.loads(f.getvalue()) + + assert isinstance(unpickled_class, type) + assert unpickled_class.__name__ == "MyClass" + assert unpickled_class.my_attribute == 1 + + +.. _pickle-oob: + +Out-of-band Buffers +------------------- + +.. versionadded:: 3.8 + +In some contexts, the :mod:`pickle` module is used to transfer massive amounts +of data. Therefore, it can be important to minimize the number of memory +copies, to preserve performance and resource consumption. However, normal +operation of the :mod:`pickle` module, as it transforms a graph-like structure +of objects into a sequential stream of bytes, intrinsically involves copying +data to and from the pickle stream. + +This constraint can be eschewed if both the *provider* (the implementation +of the object types to be transferred) and the *consumer* (the implementation +of the communications system) support the out-of-band transfer facilities +provided by pickle protocol 5 and higher. + +Provider API +^^^^^^^^^^^^ + +The large data objects to be pickled must implement a :meth:`__reduce_ex__` +method specialized for protocol 5 and higher, which returns a +:class:`PickleBuffer` instance (instead of e.g. a :class:`bytes` object) +for any large data. + +A :class:`PickleBuffer` object *signals* that the underlying buffer is +eligible for out-of-band data transfer. Those objects remain compatible +with normal usage of the :mod:`pickle` module. However, consumers can also +opt-in to tell :mod:`pickle` that they will handle those buffers by +themselves. + +Consumer API +^^^^^^^^^^^^ + +A communications system can enable custom handling of the :class:`PickleBuffer` +objects generated when serializing an object graph. + +On the sending side, it needs to pass a *buffer_callback* argument to +:class:`Pickler` (or to the :func:`dump` or :func:`dumps` function), which +will be called with each :class:`PickleBuffer` generated while pickling +the object graph. Buffers accumulated by the *buffer_callback* will not +see their data copied into the pickle stream, only a cheap marker will be +inserted. + +On the receiving side, it needs to pass a *buffers* argument to +:class:`Unpickler` (or to the :func:`load` or :func:`loads` function), +which is an iterable of the buffers which were passed to *buffer_callback*. +That iterable should produce buffers in the same order as they were passed +to *buffer_callback*. Those buffers will provide the data expected by the +reconstructors of the objects whose pickling produced the original +:class:`PickleBuffer` objects. + +Between the sending side and the receiving side, the communications system +is free to implement its own transfer mechanism for out-of-band buffers. +Potential optimizations include the use of shared memory or datatype-dependent +compression. + +Example +^^^^^^^ + +Here is a trivial example where we implement a :class:`bytearray` subclass +able to participate in out-of-band buffer pickling:: + + class ZeroCopyByteArray(bytearray): + + def __reduce_ex__(self, protocol): + if protocol >= 5: + return type(self)._reconstruct, (PickleBuffer(self),), None + else: + # PickleBuffer is forbidden with pickle protocols <= 4. + return type(self)._reconstruct, (bytearray(self),) + + @classmethod + def _reconstruct(cls, obj): + with memoryview(obj) as m: + # Get a handle over the original buffer object + obj = m.obj + if type(obj) is cls: + # Original buffer object is a ZeroCopyByteArray, return it + # as-is. + return obj + else: + return cls(obj) + +The reconstructor (the ``_reconstruct`` class method) returns the buffer's +providing object if it has the right type. This is an easy way to simulate +zero-copy behaviour on this toy example. + +On the consumer side, we can pickle those objects the usual way, which +when unserialized will give us a copy of the original object:: + + b = ZeroCopyByteArray(b"abc") + data = pickle.dumps(b, protocol=5) + new_b = pickle.loads(data) + print(b == new_b) # True + print(b is new_b) # False: a copy was made + +But if we pass a *buffer_callback* and then give back the accumulated +buffers when unserializing, we are able to get back the original object:: + + b = ZeroCopyByteArray(b"abc") + buffers = [] + data = pickle.dumps(b, protocol=5, buffer_callback=buffers.append) + new_b = pickle.loads(data, buffers=buffers) + print(b == new_b) # True + print(b is new_b) # True: no copy was made + +This example is limited by the fact that :class:`bytearray` allocates its +own memory: you cannot create a :class:`bytearray` instance that is backed +by another object's memory. However, third-party datatypes such as NumPy +arrays do not have this limitation, and allow use of zero-copy pickling +(or making as few copies as possible) when transferring between distinct +processes or systems. + +.. seealso:: :pep:`574` -- Pickle protocol 5 with out-of-band data + .. _pickle-restrict: diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 78a51573458583..3b17b9a6219870 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -68,7 +68,7 @@ support. .. class:: ImpLoader(fullname, file, filename, etc) - :term:`Loader` that wraps Python's "classic" import algorithm. + :term:`Loader ` that wraps Python's "classic" import algorithm. .. deprecated:: 3.3 This emulation is no longer needed, as the standard import mechanism @@ -227,3 +227,44 @@ support. then ``None`` is returned. In particular, the :term:`loader` for :term:`namespace packages ` does not support :meth:`get_data `. + + +.. function:: resolve_name(name) + + Resolve a name to an object. + + This functionality is used in numerous places in the standard library (see + :issue:`12915`) - and equivalent functionality is also in widely used + third-party packages such as setuptools, Django and Pyramid. + + It is expected that *name* will be a string in one of the following + formats, where W is shorthand for a valid Python identifier and dot stands + for a literal period in these pseudo-regexes: + + * ``W(.W)*`` + * ``W(.W)*:(W(.W)*)?`` + + The first form is intended for backward compatibility only. It assumes that + some part of the dotted name is a package, and the rest is an object + somewhere within that package, possibly nested inside other objects. + Because the place where the package stops and the object hierarchy starts + can't be inferred by inspection, repeated attempts to import must be done + with this form. + + In the second form, the caller makes the division point clear through the + provision of a single colon: the dotted name to the left of the colon is a + package to be imported, and the dotted name to the right is the object + hierarchy within that package. Only one import is needed in this form. If + it ends with the colon, then a module object is returned. + + The function will return an object (which might be a module), or raise one + of the following exceptions: + + :exc:`ValueError` -- if *name* isn't in a recognised format. + + :exc:`ImportError` -- if an import failed when it shouldn't have. + + :exc:`AttributeError` -- If a failure occurred when traversing the object + hierarchy within the imported package to get to the desired object. + + .. versionadded:: 3.9 diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 60c6089ad3ccb5..8e8e3775aaff4a 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -145,8 +145,8 @@ Cross Platform .. function:: system() - Returns the system/OS name, e.g. ``'Linux'``, ``'Windows'``, or ``'Java'``. An - empty string is returned if the value cannot be determined. + Returns the system/OS name, such as ``'Linux'``, ``'Darwin'``, ``'Java'``, + ``'Windows'``. An empty string is returned if the value cannot be determined. .. function:: system_alias(system, release, version) @@ -216,6 +216,21 @@ Windows Platform later (support for this was added in Python 2.6). It obviously only runs on Win32 compatible platforms. +.. function:: win32_edition() + + Returns a string representing the current Windows edition. Possible + values include but are not limited to ``'Enterprise'``, ``'IoTUAP'``, + ``'ServerStandard'``, and ``'nanoserver'``. + + .. versionadded:: 3.8 + +.. function:: win32_is_iot() + + Return ``True`` if the Windows edition returned by :func:`win32_edition` + is recognized as an IoT edition. + + .. versionadded:: 3.8 + Mac OS Platform --------------- @@ -245,4 +260,3 @@ Unix Platforms using :program:`gcc`. The file is read and scanned in chunks of *chunksize* bytes. - diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 8bd6b63a8ee50b..6def72b3736b91 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -1,8 +1,8 @@ -:mod:`plistlib` --- Generate and parse Mac OS X ``.plist`` files -================================================================ +:mod:`plistlib` --- Generate and parse Apple ``.plist`` files +============================================================= .. module:: plistlib - :synopsis: Generate and parse Mac OS X plist files. + :synopsis: Generate and parse Apple plist files. .. moduleauthor:: Jack Jansen .. sectionauthor:: Georg Brandl @@ -17,7 +17,8 @@ -------------- This module provides an interface for reading and writing the "property list" -files used mainly by Mac OS X and supports both binary and XML plist files. +files used by Apple, primarily on macOS and iOS. This module supports both binary +and XML plist files. The property list (``.plist``) file format is a simple serialization supporting basic object types, like dictionaries, lists, numbers and strings. Usually the @@ -30,12 +31,19 @@ To work with plist data in bytes objects, use :func:`dumps` and :func:`loads`. Values can be strings, integers, floats, booleans, tuples, lists, dictionaries -(but only with string keys), :class:`Data`, :class:`bytes`, :class:`bytesarray` +(but only with string keys), :class:`bytes`, :class:`bytearray` or :class:`datetime.datetime` objects. .. versionchanged:: 3.4 New API, old API deprecated. Support for binary format plists added. +.. versionchanged:: 3.8 + Support added for reading and writing :class:`UID` tokens in binary plists as used + by NSKeyedArchiver and NSKeyedUnarchiver. + +.. versionchanged:: 3.9 + Old API removed. + .. seealso:: `PList manual page `_ @@ -44,7 +52,7 @@ or :class:`datetime.datetime` objects. This module defines the following functions: -.. function:: load(fp, \*, fmt=None, use_builtin_types=True, dict_type=dict) +.. function:: load(fp, \*, fmt=None, dict_type=dict) Read a plist file. *fp* should be a readable and binary file object. Return the unpacked root object (which usually is a @@ -58,10 +66,6 @@ This module defines the following functions: * :data:`FMT_BINARY`: Binary plist format - If *use_builtin_types* is true (the default) binary data will be returned - as instances of :class:`bytes`, otherwise it is returned as instances of - :class:`Data`. - The *dict_type* is the type used for dictionaries that are read from the plist file. @@ -76,7 +80,7 @@ This module defines the following functions: .. versionadded:: 3.4 -.. function:: loads(data, \*, fmt=None, use_builtin_types=True, dict_type=dict) +.. function:: loads(data, \*, fmt=None, dict_type=dict) Load a plist from a bytes object. See :func:`load` for an explanation of the keyword arguments. @@ -120,64 +124,18 @@ This module defines the following functions: .. versionadded:: 3.4 -The following functions are deprecated: - -.. function:: readPlist(pathOrFile) - - Read a plist file. *pathOrFile* may be either a file name or a (readable - and binary) file object. Returns the unpacked root object (which usually - is a dictionary). - - This function calls :func:`load` to do the actual work, see the documentation - of :func:`that function ` for an explanation of the keyword arguments. - - .. deprecated:: 3.4 Use :func:`load` instead. - - .. versionchanged:: 3.7 - Dict values in the result are now normal dicts. You no longer can use - attribute access to access items of these dictionaries. - - -.. function:: writePlist(rootObject, pathOrFile) - - Write *rootObject* to an XML plist file. *pathOrFile* may be either a file name - or a (writable and binary) file object - - .. deprecated:: 3.4 Use :func:`dump` instead. - - -.. function:: readPlistFromBytes(data) - - Read a plist data from a bytes object. Return the root object. - - See :func:`load` for a description of the keyword arguments. - - .. deprecated:: 3.4 Use :func:`loads` instead. - - .. versionchanged:: 3.7 - Dict values in the result are now normal dicts. You no longer can use - attribute access to access items of these dictionaries. - - -.. function:: writePlistToBytes(rootObject) - - Return *rootObject* as an XML plist-formatted bytes object. - - .. deprecated:: 3.4 Use :func:`dumps` instead. - The following classes are available: -.. class:: Data(data) +.. class:: UID(data) - Return a "data" wrapper object around the bytes object *data*. This is used - in functions converting from/to plists to represent the ```` type - available in plists. + Wraps an :class:`int`. This is used when reading or writing NSKeyedArchiver + encoded data, which contains UID (see PList manual). - It has one attribute, :attr:`data`, that can be used to retrieve the Python - bytes object stored in it. + It has one attribute, :attr:`data`, which can be used to retrieve the int value + of the UID. :attr:`data` must be in the range `0 <= data < 2**64`. - .. deprecated:: 3.4 Use a :class:`bytes` object instead. + .. versionadded:: 3.8 The following constants are available: diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index d72b660d6ea9f6..2f349b35b7e0ff 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -39,6 +39,17 @@ The :mod:`poplib` module provides two classes: connection attempt (if not specified, the global default timeout setting will be used). + .. audit-event:: poplib.connect self,host,port poplib.POP3 + + .. audit-event:: poplib.putline self,line poplib.POP3 + + All commands will raise an :ref:`auditing event ` + ``poplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. + + .. versionchanged:: 3.9 + If the *timeout* parameter is set to be zero, it will raise a + :class:`ValueError` to prevent the creation of a non-blocking socket. .. class:: POP3_SSL(host, port=POP3_SSL_PORT, keyfile=None, certfile=None, timeout=None, context=None) @@ -54,6 +65,14 @@ The :mod:`poplib` module provides two classes: point to PEM-formatted private key and certificate chain files, respectively, for the SSL connection. + .. audit-event:: poplib.connect self,host,port poplib.POP3_SSL + + .. audit-event:: poplib.putline self,line popplib.POP3_SSL + + All commands will raise an :ref:`auditing event ` + ``poplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. + .. versionchanged:: 3.2 *context* parameter added. @@ -69,6 +88,10 @@ The :mod:`poplib` module provides two classes: :func:`ssl.create_default_context` select the system's trusted CA certificates for you. + .. versionchanged:: 3.9 + If the *timeout* parameter is set to be zero, it will raise a + :class:`ValueError` to prevent the creation of a non-blocking socket. + One exception is defined as an attribute of the :mod:`poplib` module: @@ -252,4 +275,3 @@ retrieves and prints all messages:: At the end of the module, there is a test section that contains a more extensive example of usage. - diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 9cbc5505aee6f3..2105af9215f769 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -44,8 +44,8 @@ by defining the relevant size and offset types as 64-bit values. Such files are sometimes referred to as :dfn:`large files`. Large file support is enabled in Python when the size of an :c:type:`off_t` is -larger than a :c:type:`long` and the :c:type:`long long` type is available and is -at least as large as an :c:type:`off_t`. +larger than a :c:type:`long` and the :c:type:`long long` is at least as large +as an :c:type:`off_t`. It may be necessary to configure and compile Python with certain compiler flags to enable this mode. For example, it is enabled by default with recent versions of Irix, but with Solaris 2.6 and 2.7 you need to do something like:: diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 9abf2865ef6143..16256c549208fc 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -25,6 +25,9 @@ width constraint. Dictionaries are sorted by key before the display is computed. +.. versionchanged:: 3.9 + Added support for pretty-printing :class:`types.SimpleNamespace`. + The :mod:`pprint` module defines one class: .. First the implementation class: @@ -144,7 +147,7 @@ The :mod:`pprint` module also provides several shortcut functions: .. index:: builtin: eval - Determine if the formatted representation of *object* is "readable," or can be + Determine if the formatted representation of *object* is "readable", or can be used to reconstruct the value using :func:`eval`. This always returns ``False`` for recursive objects. diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index d8039fd28e0d8a..34525a96f55c43 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -309,7 +309,7 @@ functions: Profile the cmd via :func:`exec` with the specified global and local environment. - .. method:: runcall(func, *args, **kwargs) + .. method:: runcall(func, /, *args, **kwargs) Profile ``func(*args, **kwargs)`` @@ -525,6 +525,17 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. ordering are identical to the :meth:`~pstats.Stats.print_callers` method. + .. method:: get_stats_profile() + + This method returns an instance of StatsProfile, which contains a mapping + of function names to instances of FunctionProfile. Each FunctionProfile + instance holds information related to the function's profile such as how + long the function took to run, how many times it was called, etc... + + .. versionadded:: 3.9 + Added the following dataclasses: StatsProfile, FunctionProfile. + Added the following function: get_stats_profile. + .. _deterministic-profiling: What Is Deterministic Profiling? diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 0ab766065d6e81..73d4f102fd4d8e 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -43,11 +43,38 @@ The :mod:`pty` module defines the following functions: Spawn a process, and connect its controlling terminal with the current process's standard io. This is often used to baffle programs which insist on - reading from the controlling terminal. - - The functions *master_read* and *stdin_read* should be functions which read from - a file descriptor. The defaults try to read 1024 bytes each time they are - called. + reading from the controlling terminal. It is expected that the process + spawned behind the pty will eventually terminate, and when it does *spawn* + will return. + + The functions *master_read* and *stdin_read* are passed a file descriptor + which they should read from, and they should always return a byte string. In + order to force spawn to return before the child process exits an + :exc:`OSError` should be thrown. + + The default implementation for both functions will read and return up to 1024 + bytes each time the function is called. The *master_read* callback is passed + the pseudoterminal’s master file descriptor to read output from the child + process, and *stdin_read* is passed file descriptor 0, to read from the + parent process's standard input. + + Returning an empty byte string from either callback is interpreted as an + end-of-file (EOF) condition, and that callback will not be called after + that. If *stdin_read* signals EOF the controlling terminal can no longer + communicate with the parent process OR the child process. Unless the child + process will quit without any input, *spawn* will then loop forever. If + *master_read* signals EOF the same behavior results (on linux at least). + + If both callbacks signal EOF then *spawn* will probably never return, unless + *select* throws an error on your platform when passed three empty lists. This + is a bug, documented in `issue 26228 `_. + + Return the exit status value from :func:`os.waitpid` on the child process. + + :func:`waitstatus_to_exitcode` can be used to convert the exit status into + an exit code. + + .. audit-event:: pty.spawn argv pty.spawn .. versionchanged:: 3.4 :func:`spawn` now returns the status value from :func:`os.waitpid` diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 8cb5a4d546c874..4fba4cba4d3568 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -27,7 +27,7 @@ byte-code cache files in the directory containing the source code. Exception raised when an error occurs while attempting to compile the file. -.. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1, invalidation_mode=PycInvalidationMode.TIMESTAMP) +.. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1, invalidation_mode=PycInvalidationMode.TIMESTAMP, quiet=0) Compile a source file to byte-code and write out the byte-code cache file. The source code is loaded from the file named *file*. The byte-code is @@ -35,13 +35,20 @@ byte-code cache files in the directory containing the source code. in ``.pyc``. For example, if *file* is ``/foo/bar/baz.py`` *cfile* will default to ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. If *dfile* is - specified, it is used as the name of the source file in error messages when + specified, it is used as the name of the source file in error messages instead of *file*. If *doraise* is true, a :exc:`PyCompileError` is raised when an error is encountered while compiling *file*. If *doraise* is false (the default), an error string is written to ``sys.stderr``, but no exception is raised. This function returns the path to byte-compiled file, i.e. whatever *cfile* value was used. + The *doraise* and *quiet* arguments determine how errors are handled while + compiling file. If *quiet* is 0 or 1, and *doraise* is false, the default + behaviour is enabled: an error string is written to ``sys.stderr``, and the + function returns ``None`` instead of a path. If *doraise* is true, + a :exc:`PyCompileError` is raised instead. However if *quiet* is 2, + no message is written, and *doraise* has no effect. + If the path that *cfile* becomes (either explicitly specified or computed) is a symlink or non-regular file, :exc:`FileExistsError` will be raised. This is to act as a warning that import will turn those paths into regular @@ -82,6 +89,9 @@ byte-code cache files in the directory containing the source code. overrides the value of the *invalidation_mode* argument, and determines its default value instead. + .. versionchanged:: 3.8 + The *quiet* parameter was added. + .. class:: PycInvalidationMode @@ -115,21 +125,33 @@ byte-code cache files in the directory containing the source code. system external to Python like a build system. -.. function:: main(args=None) +Command-Line Interface +---------------------- - Compile several source files. The files named in *args* (or on the command - line, if *args* is ``None``) are compiled and the resulting byte-code is - cached in the normal manner. This function does not search a directory - structure to locate source files; it only compiles files named explicitly. - If ``'-'`` is the only parameter in args, the list of files is taken from - standard input. +This module can be invoked as a script to compile several source +files. The files named in *filenames* are compiled and the resulting +bytecode is cached in the normal manner. This program does not search +a directory structure to locate source files; it only compiles files +named explicitly. The exit status is nonzero if one of the files could +not be compiled. - .. versionchanged:: 3.2 - Added support for ``'-'``. +.. program:: python -m py_compile + +.. cmdoption:: ... + - + + Positional arguments are files to compile. If ``-`` is the only + parameter, the list of files is taken from standard input. + +.. cmdoption:: -q, --quiet + + Suppress errors output. + +.. versionchanged:: 3.2 + Added support for ``-``. -When this module is run as a script, the :func:`main` is used to compile all the -files named on the command line. The exit status is nonzero if one of the files -could not be compiled. +.. versionchanged:: 3.10 + Added support for :option:`-q`. .. seealso:: diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index a70c8df6a7b119..36e83e85c23141 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -1,8 +1,8 @@ -:mod:`pyclbr` --- Python class browser support -============================================== +:mod:`pyclbr` --- Python module browser support +=============================================== .. module:: pyclbr - :synopsis: Supports information extraction for a Python class browser. + :synopsis: Supports information extraction for a Python module browser. .. sectionauthor:: Fred L. Drake, Jr. @@ -29,6 +29,9 @@ modules. *path* is a sequence of directory paths prepended to ``sys.path``, which is used to locate the module source code. + This function is the original interface and is only kept for back + compatibility. It returns a filtered version of the following. + .. function:: readmodule_ex(module, path=None) @@ -44,7 +47,7 @@ modules. .. versionadded:: 3.7 Descriptors for nested definitions. They are accessed through the - new children attibute. Each has a new parent attribute. + new children attribute. Each has a new parent attribute. The descriptors returned by these functions are instances of Function and Class classes. Users are not expected to create instances diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 2eeab5e2626647..0ec5900bef5bbf 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -190,32 +190,28 @@ fully processed by daemon consumer threads. Example of how to wait for enqueued tasks to be completed:: + import threading, queue + + q = queue.Queue() + def worker(): while True: item = q.get() - if item is None: - break - do_work(item) + print(f'Working on {item}') + print(f'Finished {item}') q.task_done() - q = queue.Queue() - threads = [] - for i in range(num_worker_threads): - t = threading.Thread(target=worker) - t.start() - threads.append(t) + # turn-on the worker thread + threading.Thread(target=worker, daemon=True).start() - for item in source(): + # send thirty task requests to the worker + for item in range(30): q.put(item) + print('All task requests sent\n', end='') # block until all tasks are done q.join() - - # stop workers - for i in range(num_worker_threads): - q.put(None) - for t in threads: - t.join() + print('All work completed') SimpleQueue Objects diff --git a/Doc/library/random.rst b/Doc/library/random.rst index fcedba4dbc2052..4e97b1dbad85c1 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -57,7 +57,7 @@ from sources provided by the operating system. `Complementary-Multiply-with-Carry recipe - `_ for a compatible alternative + `_ for a compatible alternative random number generator with a long period and comparatively simple update operations. @@ -86,6 +86,11 @@ Bookkeeping functions .. versionchanged:: 3.2 Moved to the version 2 scheme which uses all of the bits in a string seed. + .. deprecated:: 3.9 + In the future, the *seed* must be one of the following types: + *NoneType*, :class:`int`, :class:`float`, :class:`str`, + :class:`bytes`, or :class:`bytearray`. + .. function:: getstate() Return an object capturing the current internal state of the generator. This @@ -99,12 +104,17 @@ Bookkeeping functions the time :func:`getstate` was called. -.. function:: getrandbits(k) +Functions for bytes +------------------- - Returns a Python integer with *k* random bits. This method is supplied with - the MersenneTwister generator and some other generators may also provide it - as an optional part of the API. When available, :meth:`getrandbits` enables - :meth:`randrange` to handle arbitrarily large ranges. +.. function:: randbytes(n) + + Generate *n* random bytes. + + This method should not be used for generating security tokens. + Use :func:`secrets.token_bytes` instead. + + .. versionadded:: 3.9 Functions for integers @@ -130,6 +140,16 @@ Functions for integers Return a random integer *N* such that ``a <= N <= b``. Alias for ``randrange(a, b+1)``. +.. function:: getrandbits(k) + + Returns a Python integer with *k* random bits. This method is supplied with + the MersenneTwister generator and some other generators may also provide it + as an optional part of the API. When available, :meth:`getrandbits` enables + :meth:`randrange` to handle arbitrarily large ranges. + + .. versionchanged:: 3.9 + This method now accepts zero for *k*. + Functions for sequences ----------------------- @@ -160,7 +180,9 @@ Functions for sequences The *weights* or *cum_weights* can use any numeric type that interoperates with the :class:`float` values returned by :func:`random` (that includes - integers, floats, and fractions but excludes decimals). + integers, floats, and fractions but excludes decimals). Behavior is + undefined if any weight is negative. A :exc:`ValueError` is raised if all + weights are zero. For a given seed, the :func:`choices` function with equal weighting typically produces a different sequence than repeated calls to @@ -171,6 +193,9 @@ Functions for sequences .. versionadded:: 3.6 + .. versionchanged:: 3.9 + Raises a :exc:`ValueError` if all weights are zero. + .. function:: shuffle(x[, random]) @@ -188,8 +213,11 @@ Functions for sequences generated. For example, a sequence of length 2080 is the largest that can fit within the period of the Mersenne Twister random number generator. + .. deprecated-removed:: 3.9 3.11 + The optional parameter *random*. -.. function:: sample(population, k) + +.. function:: sample(population, k, *, counts=None) Return a *k* length list of unique elements chosen from the population sequence or set. Used for random sampling without replacement. @@ -203,6 +231,11 @@ Functions for sequences Members of the population need not be :term:`hashable` or unique. If the population contains repeats, then each occurrence is a possible selection in the sample. + Repeated elements can be specified one at a time or with the optional + keyword-only *counts* parameter. For example, ``sample(['red', 'blue'], + counts=[4, 2], k=5)`` is equivalent to ``sample(['red', 'red', 'red', 'red', + 'blue', 'blue'], k=5)``. + To choose a sample from a range of integers, use a :func:`range` object as an argument. This is especially fast and space efficient for sampling from a large population: ``sample(range(10000000), k=60)``. @@ -210,6 +243,16 @@ Functions for sequences If the sample size is larger than the population size, a :exc:`ValueError` is raised. + .. versionchanged:: 3.9 + Added the *counts* parameter. + + .. deprecated:: 3.9 + In the future, the *population* must be a sequence. Instances of + :class:`set` are no longer supported. The set must first be converted + to a :class:`list` or :class:`tuple`, preferably in a deterministic + order so that the sample is reproducible. + + Real-valued distributions ------------------------- @@ -315,6 +358,11 @@ Alternative Generator Class that implements the default pseudo-random number generator used by the :mod:`random` module. + .. deprecated:: 3.9 + In the future, the *seed* must be one of the following types: + :class:`NoneType`, :class:`int`, :class:`float`, :class:`str`, + :class:`bytes`, or :class:`bytearray`. + .. class:: SystemRandom([seed]) Class that uses the :func:`os.urandom` function for generating random numbers @@ -328,8 +376,8 @@ Alternative Generator Notes on Reproducibility ------------------------ -Sometimes it is useful to be able to reproduce the sequences given by a pseudo -random number generator. By re-using a seed value, the same sequence should be +Sometimes it is useful to be able to reproduce the sequences given by a +pseudo-random number generator. By re-using a seed value, the same sequence should be reproducible from run to run as long as multiple threads are not running. Most of the random module's algorithms and seeding functions are subject to @@ -380,12 +428,11 @@ Simulations:: >>> choices(['red', 'black', 'green'], [18, 18, 2], k=6) ['red', 'green', 'black', 'black', 'red', 'black'] - >>> # Deal 20 cards without replacement from a deck of 52 playing cards - >>> # and determine the proportion of cards with a ten-value - >>> # (a ten, jack, queen, or king). - >>> deck = collections.Counter(tens=16, low_cards=36) - >>> seen = sample(list(deck.elements()), k=20) - >>> seen.count('tens') / 20 + >>> # Deal 20 cards without replacement from a deck + >>> # of 52 playing cards, and determine the proportion of cards + >>> # with a ten-value: ten, jack, queen, or king. + >>> dealt = sample(['tens', 'low cards'], counts=[16, 36], k=20) + >>> dealt.count('tens') / 20 0.15 >>> # Estimate the probability of getting 5 or more heads from 7 spins @@ -393,29 +440,28 @@ Simulations:: >>> def trial(): ... return choices('HT', cum_weights=(0.60, 1.00), k=7).count('H') >= 5 ... - >>> sum(trial() for i in range(10000)) / 10000 + >>> sum(trial() for i in range(10_000)) / 10_000 0.4169 >>> # Probability of the median of 5 samples being in middle two quartiles >>> def trial(): - ... return 2500 <= sorted(choices(range(10000), k=5))[2] < 7500 + ... return 2_500 <= sorted(choices(range(10_000), k=5))[2] < 7_500 ... - >>> sum(trial() for i in range(10000)) / 10000 + >>> sum(trial() for i in range(10_000)) / 10_000 0.7958 Example of `statistical bootstrapping `_ using resampling -with replacement to estimate a confidence interval for the mean of a sample of -size five:: +with replacement to estimate a confidence interval for the mean of a sample:: # http://statistics.about.com/od/Applications/a/Example-Of-Bootstrapping.htm from statistics import fmean as mean from random import choices - data = 1, 2, 4, 4, 10 - means = sorted(mean(choices(data, k=5)) for i in range(20)) + data = [41, 50, 29, 37, 81, 30, 73, 63, 20, 35, 68, 22, 60, 31, 95] + means = sorted(mean(choices(data, k=len(data))) for i in range(100)) print(f'The sample mean of {mean(data):.1f} has a 90% confidence ' - f'interval from {means[1]:.1f} to {means[-2]:.1f}') + f'interval from {means[5]:.1f} to {means[94]:.1f}') Example of a `resampling permutation test `_ @@ -431,7 +477,7 @@ between the effects of a drug versus a placebo:: placebo = [54, 51, 58, 44, 55, 52, 42, 47, 58, 46] observed_diff = mean(drug) - mean(placebo) - n = 10000 + n = 10_000 count = 0 combined = drug + placebo for i in range(n): @@ -444,34 +490,31 @@ between the effects of a drug versus a placebo:: print(f'The one-sided p-value of {count / n:.4f} leads us to reject the null') print(f'hypothesis that there is no difference between the drug and the placebo.') -Simulation of arrival times and service deliveries in a single server queue:: +Simulation of arrival times and service deliveries for a multiserver queue:: + from heapq import heappush, heappop from random import expovariate, gauss - from statistics import mean, median, stdev + from statistics import mean, quantiles average_arrival_interval = 5.6 - average_service_time = 5.0 - stdev_service_time = 0.5 - - num_waiting = 0 - arrivals = [] - starts = [] - arrival = service_end = 0.0 - for i in range(20000): - if arrival <= service_end: - num_waiting += 1 - arrival += expovariate(1.0 / average_arrival_interval) - arrivals.append(arrival) - else: - num_waiting -= 1 - service_start = service_end if num_waiting else arrival - service_time = gauss(average_service_time, stdev_service_time) - service_end = service_start + service_time - starts.append(service_start) - - waits = [start - arrival for arrival, start in zip(arrivals, starts)] - print(f'Mean wait: {mean(waits):.1f}. Stdev wait: {stdev(waits):.1f}.') - print(f'Median wait: {median(waits):.1f}. Max wait: {max(waits):.1f}.') + average_service_time = 15.0 + stdev_service_time = 3.5 + num_servers = 3 + + waits = [] + arrival_time = 0.0 + servers = [0.0] * num_servers # time when each server becomes available + for i in range(100_000): + arrival_time += expovariate(1.0 / average_arrival_interval) + next_server_available = heappop(servers) + wait = max(0.0, next_server_available - arrival_time) + waits.append(wait) + service_duration = gauss(average_service_time, stdev_service_time) + service_completed = arrival_time + wait + service_duration + heappush(servers, service_completed) + + print(f'Mean wait: {mean(waits):.1f} Max wait: {max(waits):.1f}') + print('Quartiles:', [round(q, 1) for q in quantiles(waits)]) .. seealso:: diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 5ef72b535ce8d4..9abbd8ba73616e 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -540,8 +540,8 @@ character ``'$'``. Matches any character which is not a word character. This is the opposite of ``\w``. If the :const:`ASCII` flag is used this becomes the equivalent of ``[^a-zA-Z0-9_]``. If the :const:`LOCALE` flag is - used, matches characters considered alphanumeric in the current locale - and the underscore. + used, matches characters which are neither alphanumeric in the current locale + nor the underscore. .. index:: single: \Z; in regular expressions @@ -931,8 +931,8 @@ form. This is useful if you want to match an arbitrary literal string that may have regular expression metacharacters in it. For example:: - >>> print(re.escape('python.exe')) - python\.exe + >>> print(re.escape('http://www.python.org')) + http://www\.python\.org >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" >>> print('[%s]+' % re.escape(legal_chars)) @@ -942,7 +942,7 @@ form. >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) /|\-|\+|\*\*|\* - This functions must not be used for the replacement string in :func:`sub` + This function must not be used for the replacement string in :func:`sub` and :func:`subn`, only backslashes should be escaped. For example:: >>> digits_re = r'\d+' @@ -955,7 +955,9 @@ form. .. versionchanged:: 3.7 Only characters that can have special meaning in a regular expression - are escaped. + are escaped. As a result, ``'!'``, ``'"'``, ``'%'``, ``"'"``, ``','``, + ``'/'``, ``':'``, ``';'``, ``'<'``, ``'='``, ``'>'``, ``'@'``, and + ``"`"`` are no longer escaped. .. function:: purge() @@ -1615,10 +1617,14 @@ The text categories are specified with regular expressions. The technique is to combine those into a single master regular expression and to loop over successive matches:: - import collections + from typing import NamedTuple import re - Token = collections.namedtuple('Token', ['type', 'value', 'line', 'column']) + class Token(NamedTuple): + type: str + value: str + line: int + column: int def tokenize(code): keywords = {'IF', 'THEN', 'ENDIF', 'FOR', 'NEXT', 'GOSUB', 'RETURN'} diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 16c28cf7d02fb9..eae0a6df45f30e 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -19,7 +19,7 @@ function. Readline keybindings may be configured via an initialization file, typically ``.inputrc`` in your home directory. See `Readline Init File -`_ +`_ in the GNU Readline manual for information about the format and allowable constructs of that file, and the capabilities of the Readline library in general. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 3573da7ea2d716..e4eac43642d14d 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -78,6 +78,9 @@ this module for those platforms. VxWorks only supports setting :data:`RLIMIT_NOFILE`. + .. audit-event:: resource.setrlimit resource,limits resource.setrlimit + + .. function:: prlimit(pid, resource[, limits]) Combines :func:`setrlimit` and :func:`getrlimit` in one function and @@ -94,6 +97,8 @@ this module for those platforms. :exc:`PermissionError` when the user doesn't have ``CAP_SYS_RESOURCE`` for the process. + .. audit-event:: resource.prlimit pid,resource,limits resource.prlimit + .. availability:: Linux 2.6.36 or later with glibc 2.13 or later. .. versionadded:: 3.4 diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index ad96dbc95b0cd2..a4ba2848f11dde 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -20,8 +20,7 @@ scheduler: The :class:`scheduler` class defines a generic interface to scheduling events. It needs two functions to actually deal with the "outside world" --- *timefunc* should be callable without arguments, and return a number (the "time", in any - units whatsoever). If time.monotonic is not available, the *timefunc* default - is time.time instead. The *delayfunc* function should be callable with one + units whatsoever). The *delayfunc* function should be callable with one argument, compatible with the output of *timefunc*, and should delay that many time units. *delayfunc* will also be called with the argument ``0`` after each event is run to allow other threads an opportunity to run in multi-threaded @@ -105,7 +104,7 @@ Scheduler Objects .. method:: scheduler.empty() - Return true if the event queue is empty. + Return ``True`` if the event queue is empty. .. method:: scheduler.run(blocking=True) diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 733a91e20b684d..a354187c266c7f 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -117,7 +117,7 @@ The module defines the following: .. function:: select(rlist, wlist, xlist[, timeout]) This is a straightforward interface to the Unix :c:func:`select` system call. - The first three arguments are sequences of 'waitable objects': either + The first three arguments are iterables of 'waitable objects': either integers representing file descriptors or objects with a parameterless method named :meth:`~io.IOBase.fileno` returning such an integer: @@ -126,7 +126,7 @@ The module defines the following: * *xlist*: wait for an "exceptional condition" (see the manual page for what your system considers such a condition) - Empty sequences are allowed, but acceptance of three empty sequences is + Empty iterables are allowed, but acceptance of three empty iterables is platform-dependent. (It is known to work on Unix but not on Windows.) The optional *timeout* argument specifies a time-out as a floating point number in seconds. When the *timeout* argument is omitted the function blocks until @@ -141,7 +141,7 @@ The module defines the following: single: socket() (in module socket) single: popen() (in module os) - Among the acceptable object types in the sequences are Python :term:`file + Among the acceptable object types in the iterables are Python :term:`file objects ` (e.g. ``sys.stdin``, or objects returned by :func:`open` or :func:`os.popen`), socket objects returned by :func:`socket.socket`. You may also define a :dfn:`wrapper` class yourself, @@ -355,6 +355,9 @@ Edge and Level Trigger Polling (epoll) Objects Remove a registered file descriptor from the epoll object. + .. versionchanged:: 3.9 + The method no longer ignores the :data:`~errno.EBADF` error. + .. method:: epoll.poll(timeout=None, maxevents=-1) @@ -480,13 +483,14 @@ Kqueue Objects Create a kqueue object from a given file descriptor. -.. method:: kqueue.control(changelist, max_events[, timeout=None]) -> eventlist +.. method:: kqueue.control(changelist, max_events[, timeout]) -> eventlist Low level interface to kevent - - changelist must be an iterable of kevent object or ``None`` + - changelist must be an iterable of kevent objects or ``None`` - max_events must be 0 or a positive integer - - timeout in seconds (floats possible) + - timeout in seconds (floats possible); the default is ``None``, + to wait forever .. versionchanged:: 3.5 The function is now retried with a recomputed timeout when interrupted by diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f08c58179a2f9f..a94255bbf698e9 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -75,7 +75,7 @@ Two additional methods are supported: .. seealso:: - `Persistent dictionary recipe `_ + `Persistent dictionary recipe `_ with widely supported storage formats and having the speed of native dictionaries. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index fb335c69006816..7f7f0c7f124ac5 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -36,6 +36,24 @@ The :mod:`shlex` module defines the following functions: instance, passing ``None`` for *s* will read the string to split from standard input. + .. deprecated:: 3.9 + Passing ``None`` for *s* will raise an exception in future Python + versions. + +.. function:: join(split_command) + + Concatenate the tokens of the list *split_command* and return a string. + This function is the inverse of :func:`split`. + + >>> from shlex import join + >>> print(join(['echo', '-n', 'Multiple words'])) + echo -n 'Multiple words' + + The returned value is shell-escaped to protect against injection + vulnerabilities (see :func:`quote`). + + .. versionadded:: 3.8 + .. function:: quote(s) @@ -98,7 +116,9 @@ The :mod:`shlex` module defines the following class: characters, those characters will be used as the punctuation characters. Any characters in the :attr:`wordchars` attribute that appear in *punctuation_chars* will be removed from :attr:`wordchars`. See - :ref:`improved-shell-compatibility` for more information. + :ref:`improved-shell-compatibility` for more information. *punctuation_chars* + can be set only upon :class:`~shlex.shlex` instance creation and can't be + modified later. .. versionchanged:: 3.6 The *punctuation_chars* parameter was added. @@ -210,7 +230,8 @@ variables which either control lexical analysis or can be used for debugging: appear in filename specifications and command line parameters, will also be included in this attribute, and any characters which appear in ``punctuation_chars`` will be removed from ``wordchars`` if they are present - there. + there. If :attr:`whitespace_split` is set to ``True``, this will have no + effect. .. attribute:: shlex.whitespace @@ -243,11 +264,13 @@ variables which either control lexical analysis or can be used for debugging: If ``True``, tokens will only be split in whitespaces. This is useful, for example, for parsing command lines with :class:`~shlex.shlex`, getting - tokens in a similar way to shell arguments. If this attribute is ``True``, - :attr:`punctuation_chars` will have no effect, and splitting will happen - only on whitespaces. When using :attr:`punctuation_chars`, which is - intended to provide parsing closer to that implemented by shells, it is - advisable to leave ``whitespace_split`` as ``False`` (the default value). + tokens in a similar way to shell arguments. When used in combination with + :attr:`punctuation_chars`, tokens will be split on whitespace in addition to + those characters. + + .. versionchanged:: 3.8 + The :attr:`punctuation_chars` attribute was made compatible with the + :attr:`whitespace_split` attribute. .. attribute:: shlex.infile @@ -299,8 +322,8 @@ variables which either control lexical analysis or can be used for debugging: .. attribute:: shlex.punctuation_chars - Characters that will be considered punctuation. Runs of punctuation - characters will be returned as a single token. However, note that no + A read-only property. Characters that will be considered punctuation. Runs of + punctuation characters will be returned as a single token. However, note that no semantic validity checking will be performed: for example, '>>>' could be returned as a token, even though it may not be recognised as such by shells. @@ -383,12 +406,15 @@ otherwise. To illustrate, you can see the difference in the following snippet: >>> import shlex >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" - >>> list(shlex.shlex(text)) - ['a', '&', '&', 'b', ';', 'c', '&', '&', 'd', '|', '|', 'e', ';', 'f', '>', - "'abc'", ';', '(', 'def', '"ghi"', ')'] - >>> list(shlex.shlex(text, punctuation_chars=True)) - ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', "'abc'", - ';', '(', 'def', '"ghi"', ')'] + >>> s = shlex.shlex(text, posix=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] + >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', + '(', 'def', 'ghi', ')'] Of course, tokens will be returned which are not valid for shells, and you'll need to implement your own error checks on the returned tokens. @@ -413,6 +439,11 @@ which characters constitute punctuation. For example:: >>> list(s) ['~/a', '&&', 'b-c', '--color=auto', '||', 'd', '*.py?'] + However, to match the shell as closely as possible, it is recommended to + always use ``posix`` and :attr:`~shlex.whitespace_split` when using + :attr:`~shlex.punctuation_chars`, which will negate + :attr:`~shlex.wordchars` entirely. + For best effect, ``punctuation_chars`` should be set in conjunction with ``posix=True``. (Note that ``posix=False`` is the default for :class:`~shlex.shlex`.) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 2dc872fd077757..ecc3309ed5cc09 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -52,9 +52,9 @@ Directory and files operations Copy the contents (no metadata) of the file named *src* to a file named *dst* and return *dst* in the most efficient way possible. - *src* and *dst* are path names given as strings. + *src* and *dst* are path-like objects or path names given as strings. - *dst* must be the complete target file name; look at :func:`shutil.copy` + *dst* must be the complete target file name; look at :func:`~shutil.copy` for a copy that accepts a target directory path. If *src* and *dst* specify the same file, :exc:`SameFileError` is raised. @@ -67,6 +67,8 @@ Directory and files operations a new symbolic link will be created instead of copying the file *src* points to. + .. audit-event:: shutil.copyfile src,dst shutil.copyfile + .. versionchanged:: 3.3 :exc:`IOError` used to be raised instead of :exc:`OSError`. Added *follow_symlinks* argument. @@ -92,7 +94,8 @@ Directory and files operations .. function:: copymode(src, dst, *, follow_symlinks=True) Copy the permission bits from *src* to *dst*. The file contents, owner, and - group are unaffected. *src* and *dst* are path names given as strings. + group are unaffected. *src* and *dst* are path-like objects or path names + given as strings. If *follow_symlinks* is false, and both *src* and *dst* are symbolic links, :func:`copymode` will attempt to modify the mode of *dst* itself (rather than the file it points to). This functionality is not available on every @@ -100,6 +103,8 @@ Directory and files operations :func:`copymode` cannot modify symbolic links on the local platform, and it is asked to do so, it will do nothing and return. + .. audit-event:: shutil.copymode src,dst shutil.copymode + .. versionchanged:: 3.3 Added *follow_symlinks* argument. @@ -108,7 +113,8 @@ Directory and files operations Copy the permission bits, last access time, last modification time, and flags from *src* to *dst*. On Linux, :func:`copystat` also copies the "extended attributes" where possible. The file contents, owner, and - group are unaffected. *src* and *dst* are path names given as strings. + group are unaffected. *src* and *dst* are path-like objects or path + names given as strings. If *follow_symlinks* is false, and *src* and *dst* both refer to symbolic links, :func:`copystat` will operate on @@ -144,15 +150,17 @@ Directory and files operations Please see :data:`os.supports_follow_symlinks` for more information. + .. audit-event:: shutil.copystat src,dst shutil.copystat + .. versionchanged:: 3.3 Added *follow_symlinks* argument and support for Linux extended attributes. .. function:: copy(src, dst, *, follow_symlinks=True) Copies the file *src* to the file or directory *dst*. *src* and *dst* - should be strings. If *dst* specifies a directory, the file will be - copied into *dst* using the base filename from *src*. Returns the - path to the newly created file. + should be :term:`path-like objects ` or strings. If + *dst* specifies a directory, the file will be copied into *dst* using the + base filename from *src*. Returns the path to the newly created file. If *follow_symlinks* is false, and *src* is a symbolic link, *dst* will be created as a symbolic link. If *follow_symlinks* @@ -165,6 +173,10 @@ Directory and files operations To preserve all file metadata from the original, use :func:`~shutil.copy2` instead. + .. audit-event:: shutil.copyfile src,dst shutil.copy + + .. audit-event:: shutil.copymode src,dst shutil.copy + .. versionchanged:: 3.3 Added *follow_symlinks* argument. Now returns path to the newly created file. @@ -185,12 +197,17 @@ Directory and files operations However, this functionality is not available on all platforms. On platforms where some or all of this functionality is unavailable, :func:`copy2` will preserve all the metadata - it can; :func:`copy2` never returns failure. + it can; :func:`copy2` never raises an exception because it + cannot preserve file metadata. :func:`copy2` uses :func:`copystat` to copy the file metadata. Please see :func:`copystat` for more information about platform support for modifying symbolic link metadata. + .. audit-event:: shutil.copyfile src,dst shutil.copy2 + + .. audit-event:: shutil.copystat src,dst shutil.copy2 + .. versionchanged:: 3.3 Added *follow_symlinks* argument, try to copy extended file system attributes too (currently Linux only). @@ -218,7 +235,7 @@ Directory and files operations already exists. Permissions and times of directories are copied with :func:`copystat`, - individual files are copied using :func:`shutil.copy2`. + individual files are copied using :func:`~shutil.copy2`. If *symlinks* is true, symbolic links in the source tree are represented as symbolic links in the new tree and the metadata of the original links will @@ -246,8 +263,10 @@ Directory and files operations If *copy_function* is given, it must be a callable that will be used to copy each file. It will be called with the source path and the destination path - as arguments. By default, :func:`shutil.copy2` is used, but any function - that supports the same signature (like :func:`shutil.copy`) can be used. +   as arguments. By default, :func:`~shutil.copy2` is used, but any function +   that supports the same signature (like :func:`~shutil.copy`) can be used. + + .. audit-event:: shutil.copytree src,dst shutil.copytree .. versionchanged:: 3.3 Copy metadata when *symlinks* is false. @@ -296,10 +315,16 @@ Directory and files operations *excinfo*, will be the exception information returned by :func:`sys.exc_info`. Exceptions raised by *onerror* will not be caught. + .. audit-event:: shutil.rmtree path shutil.rmtree + .. versionchanged:: 3.3 Added a symlink attack resistant version that is used automatically if platform supports fd-based functions. + .. versionchanged:: 3.8 + On Windows, will no longer delete the contents of a directory junction + before removing the junction. + .. attribute:: rmtree.avoids_symlink_attacks Indicates whether the current platform and implementation provides a @@ -331,6 +356,8 @@ Directory and files operations *copy_function* allows the move to succeed when it is not possible to also copy the metadata, at the expense of not copying any of the metadata. + .. audit-event:: shutil.move src,dst shutil.move + .. versionchanged:: 3.3 Added explicit symlink handling for foreign filesystems, thus adapting it to the behavior of GNU's :program:`mv`. @@ -344,6 +371,9 @@ Directory and files operations copy the file more efficiently. See :ref:`shutil-platform-dependent-efficient-copy-operations` section. + .. versionchanged:: 3.9 + Accepts a :term:`path-like object` for both *src* and *dst*. + .. function:: disk_usage(path) Return disk usage statistics about the given path as a :term:`named tuple` @@ -367,6 +397,8 @@ Directory and files operations See also :func:`os.chown`, the underlying function. + .. audit-event:: shutil.chown path,user,group shutil.chown + .. availability:: Unix. .. versionadded:: 3.3 @@ -420,8 +452,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``". On macOS `fcopyfile`_ is used to copy the file content (not metadata). -On Linux, Solaris and other POSIX platforms where :func:`os.sendfile` supports -copies between 2 regular file descriptors :func:`os.sendfile` is used. +On Linux :func:`os.sendfile` is used. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 64 KiB) and a :func:`memoryview`-based variant of @@ -539,12 +570,14 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. available), or "xztar" (if the :mod:`lzma` module is available). *root_dir* is a directory that will be the root directory of the - archive; for example, we typically chdir into *root_dir* before creating the - archive. + archive, all paths in the archive will be relative to it; for example, + we typically chdir into *root_dir* before creating the archive. *base_dir* is the directory where we start archiving from; i.e. *base_dir* will be the common prefix of all files and - directories in the archive. + directories in the archive. *base_dir* must be given relative + to *root_dir*. See :ref:`shutil-archiving-example-with-basedir` for how to + use *base_dir* and *root_dir* together. *root_dir* and *base_dir* both default to the current directory. @@ -559,6 +592,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. The *verbose* argument is unused and deprecated. + .. audit-event:: shutil.make_archive base_name,format,root_dir,base_dir shutil.make_archive + .. versionchanged:: 3.8 The modern pax (POSIX.1-2001) format is now used instead of the legacy GNU format for archives created with ``format="tar"``. @@ -617,6 +652,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. registered for that extension. In case none is found, a :exc:`ValueError` is raised. + .. audit-event:: shutil.unpack_archive filename,extract_dir,format shutil.unpack_archive + .. versionchanged:: 3.7 Accepts a :term:`path-like object` for *filename* and *extract_dir*. @@ -692,6 +729,48 @@ The resulting archive contains: -rw-r--r-- tarek/staff 37192 2010-02-06 18:23:10 ./known_hosts +.. _shutil-archiving-example-with-basedir: + +Archiving example with *base_dir* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this example, similar to the `one above `_, +we show how to use :func:`make_archive`, but this time with the usage of +*base_dir*. We now have the following directory structure: + +.. code-block:: shell-session + + $ tree tmp + tmp + └── root + └── structure + ├── content + └── please_add.txt + └── do_not_add.txt + +In the final archive, :file:`please_add.txt` should be included, but +:file:`do_not_add.txt` should not. Therefore we use the following:: + + >>> from shutil import make_archive + >>> import os + >>> archive_name = os.path.expanduser(os.path.join('~', 'myarchive')) + >>> make_archive( + ... archive_name, + ... 'tar', + ... root_dir='tmp/root', + ... base_dir='structure/content', + ... ) + '/Users/tarek/my_archive.tar' + +Listing the files in the resulting archive gives us: + +.. code-block:: shell-session + + $ python -m tarfile -l /Users/tarek/myarchive.tar + structure/content/ + structure/content/please_add.txt + + Querying the size of the output terminal ---------------------------------------- diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index ac6cad9aff8e97..00a730b6b9ca6a 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -16,7 +16,8 @@ The :func:`signal.signal` function allows defining custom handlers to be executed when a signal is received. A small number of default handlers are installed: :const:`SIGPIPE` is ignored (so write errors on pipes and sockets can be reported as ordinary Python exceptions) and :const:`SIGINT` is -translated into a :exc:`KeyboardInterrupt` exception. +translated into a :exc:`KeyboardInterrupt` exception if the parent process +has not changed it. A handler for a particular signal, once set, remains installed until it is explicitly reset (Python emulates the BSD style interface regardless of the @@ -52,12 +53,12 @@ This has consequences: Signals and threads ^^^^^^^^^^^^^^^^^^^ -Python signal handlers are always executed in the main Python thread, +Python signal handlers are always executed in the main Python thread of the main interpreter, even if the signal was received in another thread. This means that signals can't be used as a means of inter-thread communication. You can use the synchronization primitives from the :mod:`threading` module instead. -Besides, only the main thread is allowed to set a new signal handler. +Besides, only the main thread of the main interpreter is allowed to set a new signal handler. Module contents @@ -90,11 +91,115 @@ The variables defined in the :mod:`signal` module are: signal. +.. data:: SIGABRT + + Abort signal from :manpage:`abort(3)`. + +.. data:: SIGALRM + + Timer signal from :manpage:`alarm(2)`. + + .. availability:: Unix. + +.. data:: SIGBREAK + + Interrupt from keyboard (CTRL + BREAK). + + .. availability:: Windows. + +.. data:: SIGBUS + + Bus error (bad memory access). + + .. availability:: Unix. + +.. data:: SIGCHLD + + Child process stopped or terminated. + + .. availability:: Windows. + +.. data:: SIGCLD + + Alias to :data:`SIGCHLD`. + +.. data:: SIGCONT + + Continue the process if it is currently stopped + + .. availability:: Unix. + +.. data:: SIGFPE + + Floating-point exception. For example, division by zero. + + .. seealso:: + :exc:`ZeroDivisionError` is raised when the second argument of a division + or modulo operation is zero. + +.. data:: SIGHUP + + Hangup detected on controlling terminal or death of controlling process. + + .. availability:: Unix. + +.. data:: SIGILL + + Illegal instruction. + +.. data:: SIGINT + + Interrupt from keyboard (CTRL + C). + + Default action is to raise :exc:`KeyboardInterrupt`. + +.. data:: SIGKILL + + Kill signal. + + It cannot be caught, blocked, or ignored. + + .. availability:: Unix. + +.. data:: SIGPIPE + + Broken pipe: write to pipe with no readers. + + Default action is to ignore the signal. + + .. availability:: Unix. + +.. data:: SIGSEGV + + Segmentation fault: invalid memory reference. + +.. data:: SIGTERM + + Termination signal. + +.. data:: SIGUSR1 + + User-defined signal 1. + + .. availability:: Unix. + +.. data:: SIGUSR2 + + User-defined signal 2. + + .. availability:: Unix. + +.. data:: SIGWINCH + + Window resize signal. + + .. availability:: Unix. + .. data:: SIG* All the signal numbers are defined symbolically. For example, the hangup signal is defined as :const:`signal.SIGHUP`; the variable names are identical to the - names used in C programs, as found in ````. The Unix man page for + names used in C programs, as found in ````. The Unix man page for ':c:func:`signal`' lists the existing signals (on some systems this is :manpage:`signal(2)`, on others the list is in :manpage:`signal(7)`). Note that not all systems define the same set of signal names; only those names defined by @@ -192,10 +297,10 @@ The :mod:`signal` module defines the following functions: canceled (only one alarm can be scheduled at any time). The returned value is then the number of seconds before any previously set alarm was to have been delivered. If *time* is zero, no alarm is scheduled, and any scheduled alarm is - canceled. If the return value is zero, no alarm is currently scheduled. (See - the Unix man page :manpage:`alarm(2)`.) + canceled. If the return value is zero, no alarm is currently scheduled. - .. availability:: Unix. + .. availability:: Unix. See the man page :manpage:`alarm(2)` for further + information. .. function:: getsignal(signalnum) @@ -230,8 +335,10 @@ The :mod:`signal` module defines the following functions: .. function:: pause() Cause the process to sleep until a signal is received; the appropriate handler - will then be called. Returns nothing. Not on Windows. (See the Unix man page - :manpage:`signal(2)`.) + will then be called. Returns nothing. + + .. availability:: Unix. See the man page :manpage:`signal(2)` for further + information. See also :func:`sigwait`, :func:`sigwaitinfo`, :func:`sigtimedwait` and :func:`sigpending`. @@ -244,13 +351,26 @@ The :mod:`signal` module defines the following functions: .. versionadded:: 3.8 +.. function:: pidfd_send_signal(pidfd, sig, siginfo=None, flags=0) + + Send signal *sig* to the process referred to by file descriptor *pidfd*. + Python does not currently support the *siginfo* parameter; it must be + ``None``. The *flags* argument is provided for future extensions; no flag + values are currently defined. + + See the :manpage:`pidfd_send_signal(2)` man page for more information. + + .. availability:: Linux 5.1+ + .. versionadded:: 3.9 + + .. function:: pthread_kill(thread_id, signalnum) Send the signal *signalnum* to the thread *thread_id*, another thread in the same process as the caller. The target thread can be executing any code (Python or not). However, if the target thread is executing the Python interpreter, the Python signal handlers will be :ref:`executed by the main - thread `. Therefore, the only point of sending a + thread of the main interpreter `. Therefore, the only point of sending a signal to a particular Python thread would be to force a running system call to fail with :exc:`InterruptedError`. @@ -261,8 +381,10 @@ The :mod:`signal` module defines the following functions: If *signalnum* is 0, then no signal is sent, but error checking is still performed; this can be used to check if the target thread is still running. - .. availability:: Unix (see the man page :manpage:`pthread_kill(3)` for further - information). + .. audit-event:: signal.pthread_kill thread_id,signalnum signal.pthread_kill + + .. availability:: Unix. See the man page :manpage:`pthread_kill(3)` for further + information. See also :func:`os.kill`. @@ -292,7 +414,9 @@ The :mod:`signal` module defines the following functions: For example, ``signal.pthread_sigmask(signal.SIG_BLOCK, [])`` reads the signal mask of the calling thread. - .. availability:: Unix. See the man page :manpage:`sigprocmask(3)` and + :data:`SIGKILL` and :data:`SIGSTOP` cannot be blocked. + + .. availability:: Unix. See the man page :manpage:`sigprocmask(2)` and :manpage:`pthread_sigmask(3)` for further information. See also :func:`pause`, :func:`sigpending` and :func:`sigwait`. @@ -342,7 +466,8 @@ The :mod:`signal` module defines the following functions: If not -1, *fd* must be non-blocking. It is up to the library to remove any bytes from *fd* before calling poll or select again. - When threads are enabled, this function can only be called from the main thread; + When threads are enabled, this function can only be called + from :ref:`the main thread of the main interpreter `; attempting to call it from other threads will cause a :exc:`ValueError` exception to be raised. @@ -379,8 +504,8 @@ The :mod:`signal` module defines the following functions: calls will be restarted when interrupted by signal *signalnum*, otherwise system calls will be interrupted. Returns nothing. - .. availability:: Unix (see the man page :manpage:`siginterrupt(3)` - for further information). + .. availability:: Unix. See the man page :manpage:`siginterrupt(3)` + for further information. Note that installing a signal handler with :func:`signal` will reset the restart behaviour to interruptible by implicitly calling @@ -393,9 +518,10 @@ The :mod:`signal` module defines the following functions: be a callable Python object taking two arguments (see below), or one of the special values :const:`signal.SIG_IGN` or :const:`signal.SIG_DFL`. The previous signal handler will be returned (see the description of :func:`getsignal` - above). (See the Unix man page :manpage:`signal(2)`.) + above). (See the Unix man page :manpage:`signal(2)` for further information.) - When threads are enabled, this function can only be called from the main thread; + When threads are enabled, this function can only be called + from :ref:`the main thread of the main interpreter `; attempting to call it from other threads will cause a :exc:`ValueError` exception to be raised. @@ -419,8 +545,8 @@ The :mod:`signal` module defines the following functions: thread (i.e., the signals which have been raised while blocked). Return the set of the pending signals. - .. availability:: Unix (see the man page :manpage:`sigpending(2)` for further - information). + .. availability:: Unix. See the man page :manpage:`sigpending(2)` for further + information. See also :func:`pause`, :func:`pthread_sigmask` and :func:`sigwait`. @@ -433,8 +559,8 @@ The :mod:`signal` module defines the following functions: signals specified in the signal set *sigset*. The function accepts the signal (removes it from the pending list of signals), and returns the signal number. - .. availability:: Unix (see the man page :manpage:`sigwait(3)` for further - information). + .. availability:: Unix. See the man page :manpage:`sigwait(3)` for further + information. See also :func:`pause`, :func:`pthread_sigmask`, :func:`sigpending`, :func:`sigwaitinfo` and :func:`sigtimedwait`. @@ -458,8 +584,8 @@ The :mod:`signal` module defines the following functions: :attr:`si_errno`, :attr:`si_pid`, :attr:`si_uid`, :attr:`si_status`, :attr:`si_band`. - .. availability:: Unix (see the man page :manpage:`sigwaitinfo(2)` for further - information). + .. availability:: Unix. See the man page :manpage:`sigwaitinfo(2)` for further + information. See also :func:`pause`, :func:`sigwait` and :func:`sigtimedwait`. @@ -477,8 +603,8 @@ The :mod:`signal` module defines the following functions: specifying a timeout. If *timeout* is specified as :const:`0`, a poll is performed. Returns :const:`None` if a timeout occurs. - .. availability:: Unix (see the man page :manpage:`sigtimedwait(2)` for further - information). + .. availability:: Unix. See the man page :manpage:`sigtimedwait(2)` for further + information. See also :func:`pause`, :func:`sigwait` and :func:`sigwaitinfo`. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index dfc40d17944332..b424e1ba348d87 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -8,7 +8,7 @@ -------------- -.. highlightlang:: none +.. highlight:: none **This module is automatically imported during initialization.** The automatic import can be suppressed using the interpreter's :option:`-S` option. @@ -61,6 +61,19 @@ directory rather than a file. No item is added to ``sys.path`` more than once. Blank lines and lines beginning with ``#`` are skipped. Lines starting with ``import`` (followed by space or tab) are executed. +.. note:: + + An executable line in a :file:`.pth` file is run at every Python startup, + regardless of whether a particular module is actually going to be used. + Its impact should thus be kept to a minimum. + The primary intended purpose of executable lines is to make the + corresponding module(s) importable + (load 3rd-party import hooks, adjust :envvar:`PATH` etc). + Any other initialization is supposed to be done upon a module's + actual import, if and when it happens. + Limiting a code chunk to a single line is a deliberate measure + to discourage putting anything more complex here. + .. index:: single: package triple: path; configuration; file @@ -223,6 +236,13 @@ Module contents .. versionadded:: 3.2 +.. _site-commandline: + +Command Line Interface +---------------------- + +.. program:: site + The :mod:`site` module also provides a way to get the user directories from the command line: @@ -231,8 +251,6 @@ command line: $ python3 -m site --user-site /home/user/.local/lib/python3.3/site-packages -.. program:: site - If it is called without arguments, it will print the contents of :data:`sys.path` on the standard output, followed by the value of :data:`USER_BASE` and whether the directory exists, then the same thing for diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst index 85ee8a75cf77a9..d84e74a8ceaaff 100644 --- a/Doc/library/smtpd.rst +++ b/Doc/library/smtpd.rst @@ -148,6 +148,12 @@ MailmanProxy Objects .. class:: MailmanProxy(localaddr, remoteaddr) + .. deprecated-removed:: 3.9 3.11 + + :class:`MailmanProxy` is deprecated, it depends on a ``Mailman`` + module which no longer exists and therefore is already broken. + + Create a new pure proxy server. Arguments are as per :class:`SMTPServer`. Everything will be relayed to *remoteaddr*, unless local mailman configurations knows about an address, in which case it will be handled via mailman. Note that diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 2c3a5f0c6f7296..b3cc60357f554a 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -55,6 +55,12 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). (250, b'Ok') >>> + .. audit-event:: smtplib.send self,data smtplib.SMTP + + All commands will raise an :ref:`auditing event ` + ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, + where ``data`` is the bytes about to be sent to the remote host. + .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -64,6 +70,9 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. versionadded:: 3.5 The SMTPUTF8 extension (:rfc:`6531`) is now supported. + .. versionchanged:: 3.9 + If the *timeout* parameter is set to be zero, it will raise a + :class:`ValueError` to prevent the creation of a non-blocking socket .. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, \ certfile=None [, timeout], context=None, \ @@ -102,8 +111,12 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). :func:`ssl.create_default_context` select the system's trusted CA certificates for you. + .. versionchanged:: 3.9 + If the *timeout* parameter is set to be zero, it will raise a + :class:`ValueError` to prevent the creation of a non-blocking socket -.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, source_address=None) +.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, + source_address=None[, timeout]) The LMTP protocol, which is very similar to ESMTP, is heavily based on the standard SMTP client. It's common to use Unix sockets for LMTP, so our @@ -116,6 +129,9 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). Unix socket, LMTP generally don't support or require any authentication, but your mileage might vary. + .. versionchanged:: 3.9 + The optional *timeout* parameter was added. + A nice selection of exceptions is defined as well: @@ -242,6 +258,8 @@ An :class:`SMTP` instance has the following methods: 2-tuple of the response code and message sent by the server in its connection response. + .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect + .. method:: SMTP.helo(name='') @@ -261,9 +279,10 @@ An :class:`SMTP` instance has the following methods: response for ESMTP option and store them for use by :meth:`has_extn`. Also sets several informational attributes: the message returned by the server is stored as the :attr:`ehlo_resp` attribute, :attr:`does_esmtp` - is set to true or false depending on whether the server supports ESMTP, and - :attr:`esmtp_features` will be a dictionary containing the names of the - SMTP service extensions this server supports, and their parameters (if any). + is set to ``True`` or ``False`` depending on whether the server supports + ESMTP, and :attr:`esmtp_features` will be a dictionary containing the names + of the SMTP service extensions this server supports, and their parameters + (if any). Unless you wish to use :meth:`has_extn` before sending mail, it should not be necessary to call this method explicitly. It will be implicitly called by diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst old mode 100644 new mode 100755 index 379633a3b6052b..5bcac20a4c6048 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -78,15 +78,15 @@ created. Socket addresses are represented as follows: Python programs. - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, - scopeid)`` is used, where *flowinfo* and *scopeid* represent the ``sin6_flowinfo`` + scope_id)`` is used, where *flowinfo* and *scope_id* represent the ``sin6_flowinfo`` and ``sin6_scope_id`` members in :const:`struct sockaddr_in6` in C. For - :mod:`socket` module methods, *flowinfo* and *scopeid* can be omitted just for - backward compatibility. Note, however, omission of *scopeid* can cause problems + :mod:`socket` module methods, *flowinfo* and *scope_id* can be omitted just for + backward compatibility. Note, however, omission of *scope_id* can cause problems in manipulating scoped IPv6 addresses. .. versionchanged:: 3.7 - For multicast addresses (with *scopeid* meaningful) *address* may not contain - ``%scope`` (or ``zone id``) part. This information is superfluous and may + For multicast addresses (with *scope_id* meaningful) *address* may not contain + ``%scope_id`` (or ``zone id``) part. This information is superfluous and may be safely omitted (recommended). - :const:`AF_NETLINK` sockets are represented as pairs ``(pid, groups)``. @@ -118,6 +118,10 @@ created. Socket addresses are represented as follows: - :const:`CAN_ISOTP` protocol require a tuple ``(interface, rx_addr, tx_addr)`` where both additional parameters are unsigned long integer that represent a CAN identifier (standard or extended). + - :const:`CAN_J1939` protocol require a tuple ``(interface, name, pgn, addr)`` + where additional parameters are 64-bit unsigned integer representing the + ECU name, a 32-bit unsigned integer representing the Parameter Group Number + (PGN), and an 8-bit integer representing the address. - A string or a tuple ``(id, unit)`` is used for the :const:`SYSPROTO_CONTROL` protocol of the :const:`PF_SYSTEM` family. The string is the name of a @@ -200,6 +204,23 @@ created. Socket addresses are represented as follows: .. versionadded:: 3.8 +- :const:`IPPROTO_UDPLITE` is a variant of UDP which allows you to specify + what portion of a packet is covered with the checksum. It adds two socket + options that you can change. + ``self.setsockopt(IPPROTO_UDPLITE, UDPLITE_SEND_CSCOV, length)`` will + change what portion of outgoing packets are covered by the checksum and + ``self.setsockopt(IPPROTO_UDPLITE, UDPLITE_RECV_CSCOV, length)`` will + filter out packets which cover too little of their data. In both cases + ``length`` should be in ``range(8, 2**16, 8)``. + + Such a socket should be constructed with + ``socket(AF_INET, SOCK_DGRAM, IPPROTO_UDPLITE)`` for IPv4 or + ``socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDPLITE)`` for IPv6. + + .. availability:: Linux >= 2.6.20, FreeBSD >= 10.1-RELEASE + + .. versionadded:: 3.9 + If you use a hostname in the *host* portion of IPv4/v6 socket address, the program may show a nondeterministic behavior, as Python uses the first address returned from the DNS resolution. The socket address will be resolved @@ -374,13 +395,16 @@ Constants .. availability:: Linux >= 2.6.25. + .. note:: + The :data:`CAN_BCM_CAN_FD_FRAME` flag is only available on Linux >= 4.8. + .. versionadded:: 3.4 .. data:: CAN_RAW_FD_FRAMES Enables CAN FD support in a CAN_RAW socket. This is disabled by default. This allows your application to send both CAN and CAN FD frames; however, - you one must accept both CAN and CAN FD frames when reading from the socket. + you must accept both CAN and CAN FD frames when reading from the socket. This constant is documented in the Linux documentation. @@ -388,6 +412,17 @@ Constants .. versionadded:: 3.5 +.. data:: CAN_RAW_JOIN_FILTERS + + Joins the applied CAN filters such that only CAN frames that match all + given CAN filters are passed to user space. + + This constant is documented in the Linux documentation. + + .. availability:: Linux >= 4.1. + + .. versionadded:: 3.9 + .. data:: CAN_ISOTP CAN_ISOTP, in the CAN protocol family, is the ISO-TP (ISO 15765-2) protocol. @@ -397,6 +432,15 @@ Constants .. versionadded:: 3.7 +.. data:: CAN_J1939 + + CAN_J1939, in the CAN protocol family, is the SAE J1939 protocol. + J1939 constants, documented in the Linux documentation. + + .. availability:: Linux >= 5.4. + + .. versionadded:: 3.9 + .. data:: AF_PACKET PF_PACKET @@ -513,7 +557,8 @@ The following functions all create :ref:`socket objects `. default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_`` constants. The protocol number is usually zero and may be omitted or in the case where the address family is :const:`AF_CAN` the protocol - should be one of :const:`CAN_RAW`, :const:`CAN_BCM` or :const:`CAN_ISOTP`. + should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or + :const:`CAN_J1939`. If *fileno* is specified, the values for *family*, *type*, and *proto* are auto-detected from the specified file descriptor. Auto-detection can be @@ -526,6 +571,8 @@ The following functions all create :ref:`socket objects `. The newly created socket is :ref:`non-inheritable `. + .. audit-event:: socket.__new__ self,family,type,protocol socket.socket + .. versionchanged:: 3.3 The AF_CAN family was added. The AF_RDS family was added. @@ -543,7 +590,9 @@ The following functions all create :ref:`socket objects `. When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` bit flags are applied to *type* they are cleared, and :attr:`socket.type` will not reflect them. They are still passed - to the underlying system `socket()` call. Therefore:: + to the underlying system `socket()` call. Therefore, + + :: sock = socket.socket( socket.AF_INET, @@ -553,6 +602,9 @@ The following functions all create :ref:`socket objects `. ``SOCK_NONBLOCK``, but ``sock.type`` will be set to ``socket.SOCK_STREAM``. + .. versionchanged:: 3.9 + The CAN_J1939 protocol was added. + .. function:: socketpair([family[, type[, proto]]]) Build a pair of connected socket objects using the given address family, socket @@ -714,10 +766,12 @@ The :mod:`socket` module also offers various network-related services: :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose format depends on the returned *family* (a ``(address, port)`` 2-tuple for - :const:`AF_INET`, a ``(address, port, flow info, scope id)`` 4-tuple for + :const:`AF_INET`, a ``(address, port, flowinfo, scope_id)`` 4-tuple for :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` method. + .. audit-event:: socket.getaddrinfo host,port,family,type,protocol socket.getaddrinfo + The following example fetches address information for a hypothetical TCP connection to ``example.org`` on port 80 (results may differ on your system if IPv6 isn't enabled):: @@ -733,7 +787,7 @@ The :mod:`socket` module also offers various network-related services: .. versionchanged:: 3.7 for IPv6 multicast addresses, string representing an address will not - contain ``%scope`` part. + contain ``%scope_id`` part. .. function:: getfqdn([name]) @@ -753,6 +807,8 @@ The :mod:`socket` module also offers various network-related services: interface. :func:`gethostbyname` does not support IPv6 name resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. + .. audit-event:: socket.gethostbyname hostname socket.gethostbyname + .. function:: gethostbyname_ex(hostname) @@ -765,12 +821,16 @@ The :mod:`socket` module also offers various network-related services: resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. + .. audit-event:: socket.gethostbyname hostname socket.gethostbyname_ex + .. function:: gethostname() Return a string containing the hostname of the machine where the Python interpreter is currently executing. + .. audit-event:: socket.gethostname "" socket.gethostname + Note: :func:`gethostname` doesn't always return the fully qualified domain name; use :func:`getfqdn` for that. @@ -785,6 +845,8 @@ The :mod:`socket` module also offers various network-related services: domain name, use the function :func:`getfqdn`. :func:`gethostbyaddr` supports both IPv4 and IPv6. + .. audit-event:: socket.gethostbyaddr ip_address socket.gethostbyaddr + .. function:: getnameinfo(sockaddr, flags) @@ -793,11 +855,13 @@ The :mod:`socket` module also offers various network-related services: or numeric address representation in *host*. Similarly, *port* can contain a string port name or a numeric port number. - For IPv6 addresses, ``%scope`` is appended to the host part if *sockaddr* - contains meaningful *scopeid*. Usually this happens for multicast addresses. + For IPv6 addresses, ``%scope_id`` is appended to the host part if *sockaddr* + contains meaningful *scope_id*. Usually this happens for multicast addresses. For more information about *flags* you can consult :manpage:`getnameinfo(3)`. + .. audit-event:: socket.getnameinfo sockaddr socket.getnameinfo + .. function:: getprotobyname(protocolname) Translate an Internet protocol name (for example, ``'icmp'``) to a constant @@ -813,6 +877,8 @@ The :mod:`socket` module also offers various network-related services: service. The optional protocol name, if given, should be ``'tcp'`` or ``'udp'``, otherwise any protocol will match. + .. audit-event:: socket.getservbyname servicename,protocolname socket.getservbyname + .. function:: getservbyport(port[, protocolname]) @@ -820,6 +886,8 @@ The :mod:`socket` module also offers various network-related services: service. The optional protocol name, if given, should be ``'tcp'`` or ``'udp'``, otherwise any protocol will match. + .. audit-event:: socket.getservbyport port,protocolname socket.getservbyport + .. function:: ntohl(x) @@ -1003,6 +1071,8 @@ The :mod:`socket` module also offers various network-related services: Set the machine's hostname to *name*. This will raise an :exc:`OSError` if you don't have enough rights. + .. audit-event:: socket.sethostname name socket.sethostname + .. availability:: Unix. .. versionadded:: 3.3 @@ -1014,10 +1084,13 @@ The :mod:`socket` module also offers various network-related services: (index int, name string) tuples. :exc:`OSError` if the system call fails. - .. availability:: Unix. + .. availability:: Unix, Windows. .. versionadded:: 3.3 + .. versionchanged:: 3.8 + Windows support was added. + .. function:: if_nametoindex(if_name) @@ -1025,10 +1098,13 @@ The :mod:`socket` module also offers various network-related services: interface name. :exc:`OSError` if no interface with the given name exists. - .. availability:: Unix. + .. availability:: Unix, Windows. .. versionadded:: 3.3 + .. versionchanged:: 3.8 + Windows support was added. + .. function:: if_indextoname(if_index) @@ -1036,10 +1112,13 @@ The :mod:`socket` module also offers various network-related services: interface index number. :exc:`OSError` if no interface with the given index exists. - .. availability:: Unix. + .. availability:: Unix, Windows. .. versionadded:: 3.3 + .. versionchanged:: 3.8 + Windows support was added. + .. _socket-objects: @@ -1078,6 +1157,7 @@ to sockets. Bind the socket to *address*. The socket must not already be bound. (The format of *address* depends on the address family --- see above.) + .. audit-event:: socket.bind self,address socket.socket.bind .. method:: socket.close() @@ -1115,6 +1195,8 @@ to sockets. :exc:`InterruptedError` exception if the connection is interrupted by a signal (or the exception raised by the signal handler). + .. audit-event:: socket.connect self,address socket.socket.connect + .. versionchanged:: 3.5 The method now waits until the connection completes instead of raising an :exc:`InterruptedError` exception if the connection is interrupted by a @@ -1131,6 +1213,7 @@ to sockets. :c:data:`errno` variable. This is useful to support, for example, asynchronous connects. + .. audit-event:: socket.connect self,address socket.socket.connect_ex .. method:: socket.detach() @@ -1299,7 +1382,7 @@ to sockets. .. versionchanged:: 3.7 For multicast IPv6 address, first item of *address* does not contain - ``%scope`` part anymore. In order to get full IPv6 address use + ``%scope_id`` part anymore. In order to get full IPv6 address use :func:`getnameinfo`. .. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]]) @@ -1358,9 +1441,9 @@ to sockets. fds = array.array("i") # Array of ints msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) for cmsg_level, cmsg_type, cmsg_data in ancdata: - if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS): + if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: # Append data, ignoring any truncated integers at the end. - fds.fromstring(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) return msg, list(fds) .. availability:: most Unix platforms, possibly others. @@ -1472,6 +1555,8 @@ to sockets. bytes sent. (The format of *address* depends on the address family --- see above.) + .. audit-event:: socket.sendto self,address socket.socket.sendto + .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an exception, the method now retries the system call instead of raising @@ -1511,6 +1596,8 @@ to sockets. .. availability:: most Unix platforms, possibly others. + .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg + .. versionadded:: 3.3 .. versionchanged:: 3.5 @@ -1527,6 +1614,29 @@ to sockets. .. versionadded:: 3.6 +.. method:: socket.send_fds(sock, buffers, fds[, flags[, address]]) + + Send the list of file descriptors *fds* over an :const:`AF_UNIX` socket. + The *fds* parameter is a sequence of file descriptors. + Consult :meth:`sendmsg` for the documentation of these parameters. + + .. availability:: Unix supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. + + .. versionadded:: 3.9 + +.. method:: socket.recv_fds(sock, bufsize, maxfds[, flags]) + + Receive up to *maxfds* file descriptors. Return ``(msg, list(fds), flags, addr)``. Consult + :meth:`recvmsg` for the documentation of these parameters. + + .. availability:: Unix supporting :meth:`~socket.recvmsg` and :const:`SCM_RIGHTS` mechanism. + + .. versionadded:: 3.9 + + .. note:: + + Any truncated integers at the end of the list of file descriptors. + .. method:: socket.sendfile(file, offset=0, count=None) Send a file until EOF is reached by using high-performance @@ -1585,7 +1695,9 @@ to sockets. .. method:: socket.setsockopt(level, optname, value: int) .. method:: socket.setsockopt(level, optname, value: buffer) + :noindex: .. method:: socket.setsockopt(level, optname, None, optlen: int) + :noindex: .. index:: module: struct @@ -1595,9 +1707,9 @@ to sockets. ``None`` or a :term:`bytes-like object` representing a buffer. In the later case it is up to the caller to ensure that the bytestring contains the proper bits (see the optional built-in module :mod:`struct` for a way to - encode C structures as bytestrings). When value is set to ``None``, - optlen argument is required. It's equivalent to call setsockopt C - function with optval=NULL and optlen=optlen. + encode C structures as bytestrings). When *value* is set to ``None``, + *optlen* argument is required. It's equivalent to call :c:func:`setsockopt` C + function with ``optval=NULL`` and ``optlen=optlen``. .. versionchanged:: 3.5 diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 7c8c8d52e03d95..232c0616d925dc 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -237,6 +237,8 @@ Server Objects .. method:: shutdown() Tell the :meth:`serve_forever` loop to stop and wait until it does. + :meth:`shutdown` must be called while :meth:`serve_forever` is running in a + different thread otherwise it will deadlock. .. method:: server_close() diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 37087ac5af492a..13aa8c512d0319 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -18,7 +18,8 @@ application using SQLite and then port the code to a larger database such as PostgreSQL or Oracle. The sqlite3 module was written by Gerhard Häring. It provides a SQL interface -compliant with the DB-API 2.0 specification described by :pep:`249`. +compliant with the DB-API 2.0 specification described by :pep:`249`, and +requires SQLite 3.7.3 or newer. To use the module, you must first create a :class:`Connection` object that represents the database. Here the data will be stored in the @@ -165,9 +166,10 @@ Module functions and constants that 'mytype' is the type of the column. It will try to find an entry of 'mytype' in the converters dictionary and then use the converter function found there to return the value. The column name found in :attr:`Cursor.description` - is only the first word of the column name, i. e. if you use something like - ``'as "x [datetime]"'`` in your SQL, then we will parse out everything until the - first blank for the column name: the column name would simply be "x". + does not include the type, i. e. if you use something like + ``'as "Expiration date [datetime]"'`` in your SQL, then we will parse out + everything until the first ``'['`` for the column name and strip + the preceeding space: the column name would simply be "Expiration date". .. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) @@ -224,6 +226,8 @@ Module functions and constants More information about this feature, including a list of recognized options, can be found in the `SQLite URI documentation `_. + .. audit-event:: sqlite3.connect database sqlite3.connect + .. versionchanged:: 3.4 Added the *uri* parameter. @@ -537,6 +541,7 @@ Connection Objects with open('dump.sql', 'w') as f: for line in con.iterdump(): f.write('%s\n' % line) + con.close() .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) @@ -573,8 +578,11 @@ Connection Objects print(f'Copied {total-remaining} of {total} pages...') con = sqlite3.connect('existing_db.db') - with sqlite3.connect('backup.db') as bck: + bck = sqlite3.connect('backup.db') + with bck: con.backup(bck, pages=1, progress=progress) + bck.close() + con.close() Example 2, copy an existing database into a transient copy:: @@ -584,8 +592,6 @@ Connection Objects dest = sqlite3.connect(':memory:') source.backup(dest) - Availability: SQLite 3.6.11 or higher - .. versionadded:: 3.7 @@ -694,9 +700,6 @@ Cursor Objects statements because we cannot determine the number of rows a query produced until all rows were fetched. - With SQLite versions before 3.6.5, :attr:`rowcount` is set to 0 if - you make a ``DELETE FROM table`` without any condition. - .. attribute:: lastrowid This read-only attribute provides the rowid of the last modified row. It is @@ -921,7 +924,7 @@ a class like this:: self.x, self.y = x, y Now you want to store the point in a single SQLite column. First you'll have to -choose one of the supported types first to be used for representing the point. +choose one of the supported types to be used for representing the point. Let's just use str and separate the coordinates using a semicolon. Then you need to give your class a method ``__conform__(self, protocol)`` which must return the converted value. The parameter *protocol* will be :class:`PrepareProtocol`. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 6a441983f8884e..1cfd165202d0ef 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -42,7 +42,7 @@ This module provides a class, :class:`ssl.SSLSocket`, which is derived from the :class:`socket.socket` type, and provides a socket-like wrapper that also encrypts and decrypts the data going over the socket with SSL. It supports additional methods such as :meth:`getpeercert`, which retrieves the -certificate of the other side of the connection, and :meth:`cipher`,which +certificate of the other side of the connection, and :meth:`cipher`, which retrieves the cipher being used for the secure connection. For more sophisticated applications, the :class:`ssl.SSLContext` class @@ -139,6 +139,10 @@ purposes. *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load default CA certificates. + When :attr:`~SSLContext.keylog_filename` is supported and the environment + variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` + enables key logging. + .. note:: The protocol, options, cipher and other settings may change to more restrictive values anytime without prior deprecation. The values @@ -172,6 +176,10 @@ purposes. 3DES was dropped from the default cipher string. + .. versionchanged:: 3.8 + + Support for key logging to :envvar:`SSLKEYLOGFILE` was added. + Exceptions ^^^^^^^^^^ @@ -285,7 +293,7 @@ Random generation Read the Wikipedia article, `Cryptographically secure pseudorandom number generator (CSPRNG) `_, - to get the requirements of a cryptographically generator. + to get the requirements of a cryptographically strong generator. .. versionadded:: 3.3 @@ -605,9 +613,9 @@ Constants .. data:: VERIFY_CRL_CHECK_LEAF Possible value for :attr:`SSLContext.verify_flags`. In this mode, only the - peer cert is check but non of the intermediate CA certificates. The mode + peer cert is checked but none of the intermediate CA certificates. The mode requires a valid CRL that is signed by the peer cert's issuer (its direct - ancestor CA). If no proper has been loaded + ancestor CA). If no proper CRL has been loaded with :attr:`SSLContext.load_verify_locations`, validation will fail. .. versionadded:: 3.4 @@ -665,7 +673,7 @@ Constants .. data:: PROTOCOL_SSLv23 - Alias for data:`PROTOCOL_TLS`. + Alias for :data:`PROTOCOL_TLS`. .. deprecated:: 3.6 @@ -1056,6 +1064,7 @@ Constants SSL 3.0 to TLS 1.3. + SSL Sockets ----------- @@ -1247,6 +1256,9 @@ SSL sockets also have the following additional methods and attributes: The returned dictionary includes additional X509v3 extension items such as ``crlDistributionPoints``, ``caIssuers`` and ``OCSP`` URIs. + .. versionchanged:: 3.9 + IPv6 address strings no longer have a trailing new line. + .. method:: SSLSocket.cipher() Returns a three-value tuple containing the name of the cipher being used, the @@ -1649,7 +1661,7 @@ to speed up repeated connections from the same clients. return the agreed-upon protocol. This method will raise :exc:`NotImplementedError` if :data:`HAS_ALPN` is - False. + ``False``. OpenSSL 1.1.0 to 1.1.0e will abort the handshake and raise :exc:`SSLError` when both sides support ALPN but cannot agree on a protocol. 1.1.0f+ @@ -1668,7 +1680,7 @@ to speed up repeated connections from the same clients. return the agreed-upon protocol. This method will raise :exc:`NotImplementedError` if :data:`HAS_NPN` is - False. + ``False``. .. versionadded:: 3.3 @@ -1821,7 +1833,7 @@ to speed up repeated connections from the same clients. .. attribute:: SSLContext.sslsocket_class - The return type of :meth:`SSLContext.wrap_sockets`, defaults to + The return type of :meth:`SSLContext.wrap_socket`, defaults to :class:`SSLSocket`. The attribute can be overridden on instance of class in order to return a custom subclass of :class:`SSLSocket`. @@ -1831,7 +1843,7 @@ to speed up repeated connections from the same clients. server_hostname=None, session=None) Wrap the BIO objects *incoming* and *outgoing* and return an instance of - attr:`SSLContext.sslobject_class` (default :class:`SSLObject`). The SSL + :attr:`SSLContext.sslobject_class` (default :class:`SSLObject`). The SSL routines will read input data from the incoming BIO and write data to the outgoing BIO. @@ -1873,13 +1885,15 @@ to speed up repeated connections from the same clients. :meth:`~SSLContext.wrap_socket` in order to match the hostname. Enabling hostname checking automatically sets :attr:`~SSLContext.verify_mode` from :data:`CERT_NONE` to :data:`CERT_REQUIRED`. It cannot be set back to - :data:`CERT_NONE` as long as hostname checking is enabled. + :data:`CERT_NONE` as long as hostname checking is enabled. The + :data:`PROTOCOL_TLS_CLIENT` protocol enables hostname checking by default. + With other protocols, hostname checking must be enabled explicitly. Example:: import socket, ssl - context = ssl.SSLContext() + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) context.verify_mode = ssl.CERT_REQUIRED context.check_hostname = True context.load_default_certs() @@ -1901,6 +1915,20 @@ to speed up repeated connections from the same clients. This features requires OpenSSL 0.9.8f or newer. +.. attribute:: SSLContext.keylog_filename + + Write TLS keys to a keylog file, whenever key material is generated or + received. The keylog file is designed for debugging purposes only. The + file format is specified by NSS and used by many traffic analyzers such + as Wireshark. The log file is opened in append-only mode. Writes are + synchronized between threads, but not between processes. + + .. versionadded:: 3.8 + + .. note:: + + This features requires OpenSSL 1.1.1 or newer. + .. attribute:: SSLContext.maximum_version A :class:`TLSVersion` enum member representing the highest supported @@ -1936,6 +1964,19 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.7 +.. attribute:: SSLContext.num_tickets + + Control the number of TLS 1.3 session tickets of a + :attr:`TLS_PROTOCOL_SERVER` context. The setting has no impact on TLS + 1.0 to 1.2 connections. + + .. note:: + + This attribute is not available unless the ssl module is compiled + with OpenSSL 1.1.1 or newer. + + .. versionadded:: 3.8 + .. attribute:: SSLContext.options An integer representing the set of SSL options enabled on this context. @@ -1991,6 +2032,16 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.7 +.. attribute:: SSLContext.security_level + + An integer representing the `security level + `_ + for the context. This attribute is read-only. + + .. availability:: OpenSSL 1.1.0 or newer + + .. versionadded:: 3.10 + .. attribute:: SSLContext.verify_flags The flags for certificate verification operations. You can set flags like @@ -2181,19 +2232,23 @@ If you prefer to tune security settings yourself, you might create a context from scratch (but beware that you might not get the settings right):: - >>> context = ssl.SSLContext() - >>> context.verify_mode = ssl.CERT_REQUIRED - >>> context.check_hostname = True + >>> context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) >>> context.load_verify_locations("/etc/ssl/certs/ca-bundle.crt") (this snippet assumes your operating system places a bundle of all CA certificates in ``/etc/ssl/certs/ca-bundle.crt``; if not, you'll get an error and have to adjust the location) +The :data:`PROTOCOL_TLS_CLIENT` protocol configures the context for cert +validation and hostname verification. :attr:`~SSLContext.verify_mode` is +set to :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` is set +to ``True``. All other protocols create SSL contexts with insecure defaults. + When you use the context to connect to a server, :const:`CERT_REQUIRED` -validates the server certificate: it ensures that the server certificate -was signed with one of the CA certificates, and checks the signature for -correctness:: +and :attr:`~SSLContext.check_hostname` validate the server certificate: it +ensures that the server certificate was signed with one of the CA +certificates, checks the signature for correctness, and verifies other +properties like validity and identity of the hostname:: >>> conn = context.wrap_socket(socket.socket(socket.AF_INET), ... server_hostname="www.python.org") @@ -2226,7 +2281,7 @@ Visual inspection shows that the certificate does identify the desired service (('postalCode', '03894-4801'),), (('countryName', 'US'),), (('stateOrProvinceName', 'NH'),), - (('localityName', 'Wolfeboro,'),), + (('localityName', 'Wolfeboro'),), (('organizationName', 'Python Software Foundation'),), (('commonName', 'www.python.org'),)), 'subjectAltName': (('DNS', 'www.python.org'), @@ -2441,14 +2496,17 @@ provided. - :meth:`~SSLSocket.read` - :meth:`~SSLSocket.write` - :meth:`~SSLSocket.getpeercert` + - :meth:`~SSLSocket.selected_alpn_protocol` - :meth:`~SSLSocket.selected_npn_protocol` - :meth:`~SSLSocket.cipher` - :meth:`~SSLSocket.shared_ciphers` - :meth:`~SSLSocket.compression` - :meth:`~SSLSocket.pending` - :meth:`~SSLSocket.do_handshake` + - :meth:`~SSLSocket.verify_client_post_handshake` - :meth:`~SSLSocket.unwrap` - :meth:`~SSLSocket.get_channel_binding` + - :meth:`~SSLSocket.version` When compared to :class:`SSLSocket`, this object lacks the following features: diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index c8f6904f9b1f7a..f48a0a9faa6b0c 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -425,3 +425,13 @@ for more detail on the meaning of these constants. FILE_ATTRIBUTE_VIRTUAL .. versionadded:: 3.5 + +On Windows, the following constants are available for comparing against the +``st_reparse_tag`` member returned by :func:`os.lstat`. These are well-known +constants, but are not an exhaustive list. + +.. data:: IO_REPARSE_TAG_SYMLINK + IO_REPARSE_TAG_MOUNT_POINT + IO_REPARSE_TAG_APPEXECLINK + + .. versionadded:: 3.8 diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 8bb2bdf7b697de..38a499ab37e890 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -2,7 +2,7 @@ ======================================================= .. module:: statistics - :synopsis: mathematical statistics functions + :synopsis: Mathematical statistics functions .. moduleauthor:: Steven D'Aprano .. sectionauthor:: Steven D'Aprano @@ -19,17 +19,21 @@ -------------- This module provides functions for calculating mathematical statistics of -numeric (:class:`Real`-valued) data. - -.. note:: - - Unless explicitly noted otherwise, these functions support :class:`int`, - :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`. - Behaviour with other types (whether in the numeric tower or not) is - currently unsupported. Mixed types are also undefined and - implementation-dependent. If your input data consists of mixed types, - you may be able to use :func:`map` to ensure a consistent result, e.g. - ``map(float, input_data)``. +numeric (:class:`~numbers.Real`-valued) data. + +The module is not intended to be a competitor to third-party libraries such +as `NumPy `_, `SciPy `_, or +proprietary full-featured statistics packages aimed at professional +statisticians such as Minitab, SAS and Matlab. It is aimed at the level of +graphing and scientific calculators. + +Unless explicitly noted, these functions support :class:`int`, +:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. +Behaviour with other types (whether in the numeric tower or not) is +currently unsupported. Collections with a mix of types are also undefined +and implementation-dependent. If your input data consists of mixed types, +you may be able to use :func:`map` to ensure a consistent result, for +example: ``map(float, input_data)``. Averages and measures of central location ----------------------------------------- @@ -48,6 +52,7 @@ or sample. :func:`median_grouped` Median, or 50th percentile, of grouped data. :func:`mode` Single mode (most common value) of discrete or nominal data. :func:`multimode` List of modes (most common values) of discrete or nomimal data. +:func:`quantiles` Divide data into intervals with equal probability. ======================= =============================================================== Measures of spread @@ -72,7 +77,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: mean(data) - Return the sample arithmetic mean of *data* which can be a sequence or iterator. + Return the sample arithmetic mean of *data* which can be a sequence or iterable. The arithmetic mean is the sum of the data divided by the number of data points. It is commonly called "the average", although it is only one of many @@ -101,14 +106,12 @@ However, for reading convenience, most of the examples show sorted sequences. .. note:: The mean is strongly affected by outliers and is not a robust estimator - for central location: the mean is not necessarily a typical example of the - data points. For more robust, although less efficient, measures of - central location, see :func:`median` and :func:`mode`. (In this case, - "efficient" refers to statistical efficiency rather than computational - efficiency.) + for central location: the mean is not necessarily a typical example of + the data points. For more robust measures of central location, see + :func:`median` and :func:`mode`. The sample mean gives an unbiased estimate of the true population mean, - which means that, taken on average over all the possible samples, + so that when taken on average over all the possible samples, ``mean(sample)`` converges on the true mean of the entire population. If *data* represents the entire population rather than a sample, then ``mean(data)`` is equivalent to calculating the true population mean μ. @@ -119,9 +122,8 @@ However, for reading convenience, most of the examples show sorted sequences. Convert *data* to floats and compute the arithmetic mean. This runs faster than the :func:`mean` function and it always returns a - :class:`float`. The result is highly accurate but not as perfect as - :func:`mean`. If the input dataset is empty, raises a - :exc:`StatisticsError`. + :class:`float`. The *data* may be a sequence or iterable. If the input + dataset is empty, raises a :exc:`StatisticsError`. .. doctest:: @@ -135,15 +137,20 @@ However, for reading convenience, most of the examples show sorted sequences. Convert *data* to floats and compute the geometric mean. + The geometric mean indicates the central tendency or typical value of the + *data* using the product of the values (as opposed to the arithmetic mean + which uses their sum). + Raises a :exc:`StatisticsError` if the input dataset is empty, if it contains a zero, or if it contains a negative value. + The *data* may be a sequence or iterable. No special efforts are made to achieve exact results. (However, this may change in the future.) .. doctest:: - >>> round(geometric_mean([54, 24, 36]), 9) + >>> round(geometric_mean([54, 24, 36]), 1) 36.0 .. versionadded:: 3.8 @@ -151,17 +158,26 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: harmonic_mean(data) - Return the harmonic mean of *data*, a sequence or iterator of + Return the harmonic mean of *data*, a sequence or iterable of real-valued numbers. The harmonic mean, sometimes called the subcontrary mean, is the reciprocal of the arithmetic :func:`mean` of the reciprocals of the data. For example, the harmonic mean of three values *a*, *b* and *c* - will be equivalent to ``3/(1/a + 1/b + 1/c)``. + will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the values + is zero, the result will be zero. The harmonic mean is a type of average, a measure of the central - location of the data. It is often appropriate when averaging quantities - which are rates or ratios, for example speeds. For example: + location of the data. It is often appropriate when averaging + rates or ratios, for example speeds. + + Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr. + What is the average speed? + + .. doctest:: + + >>> harmonic_mean([40, 60]) + 48.0 Suppose an investor purchases an equal value of shares in each of three companies, with P/E (price/earning) ratios of 2.5, 3 and 10. @@ -172,12 +188,13 @@ However, for reading convenience, most of the examples show sorted sequences. >>> harmonic_mean([2.5, 3, 10]) # For an equal investment portfolio. 3.6 - Using the arithmetic mean would give an average of about 5.167, which - is too high. - :exc:`StatisticsError` is raised if *data* is empty, or any element is less than zero. + The current algorithm has an early-out when it encounters a zero + in the input. This means that the subsequent inputs are not tested + for validity. (This behavior may change in the future.) + .. versionadded:: 3.6 @@ -185,11 +202,11 @@ However, for reading convenience, most of the examples show sorted sequences. Return the median (middle value) of numeric data, using the common "mean of middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. - *data* can be a sequence or iterator. + *data* can be a sequence or iterable. - The median is a robust measure of central location, and is less affected by - the presence of outliers in your data. When the number of data points is - odd, the middle data point is returned: + The median is a robust measure of central location and is less affected by + the presence of outliers. When the number of data points is odd, the + middle data point is returned: .. doctest:: @@ -207,17 +224,14 @@ However, for reading convenience, most of the examples show sorted sequences. This is suited for when your data is discrete, and you don't mind that the median may not be an actual data point. - If your data is ordinal (supports order operations) but not numeric (doesn't - support addition), you should use :func:`median_low` or :func:`median_high` + If the data is ordinal (supports order operations) but not numeric (doesn't + support addition), consider using :func:`median_low` or :func:`median_high` instead. - .. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped` - - .. function:: median_low(data) Return the low median of numeric data. If *data* is empty, - :exc:`StatisticsError` is raised. *data* can be a sequence or iterator. + :exc:`StatisticsError` is raised. *data* can be a sequence or iterable. The low median is always a member of the data set. When the number of data points is odd, the middle value is returned. When it is even, the smaller of @@ -237,7 +251,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: median_high(data) Return the high median of data. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterator. + is raised. *data* can be a sequence or iterable. The high median is always a member of the data set. When the number of data points is odd, the middle value is returned. When it is even, the larger of @@ -258,7 +272,7 @@ However, for reading convenience, most of the examples show sorted sequences. Return the median of grouped continuous data, calculated as the 50th percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterator. + is raised. *data* can be a sequence or iterable. .. doctest:: @@ -311,10 +325,12 @@ However, for reading convenience, most of the examples show sorted sequences. The mode (when it exists) is the most typical value and serves as a measure of central location. - If there are multiple modes, returns the first one encountered in the *data*. - If *data* is empty, :exc:`StatisticsError` is raised. + If there are multiple modes with the same frequency, returns the first one + encountered in the *data*. If the smallest or largest of those is + desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``. + If the input *data* is empty, :exc:`StatisticsError` is raised. - ``mode`` assumes discrete data, and returns a single value. This is the + ``mode`` assumes discrete data and returns a single value. This is the standard treatment of the mode as commonly taught in schools: .. doctest:: @@ -322,8 +338,8 @@ However, for reading convenience, most of the examples show sorted sequences. >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) 3 - The mode is unique in that it is the only statistic which also applies - to nominal (non-numeric) data: + The mode is unique in that it is the only statistic in this package that + also applies to nominal (non-numeric) data: .. doctest:: @@ -365,15 +381,16 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: pvariance(data, mu=None) - Return the population variance of *data*, a non-empty iterable of real-valued - numbers. Variance, or second moment about the mean, is a measure of the - variability (spread or dispersion) of data. A large variance indicates that - the data is spread out; a small variance indicates it is clustered closely - around the mean. + Return the population variance of *data*, a non-empty sequence or iterable + of real-valued numbers. Variance, or second moment about the mean, is a + measure of the variability (spread or dispersion) of data. A large + variance indicates that the data is spread out; a small variance indicates + it is clustered closely around the mean. - If the optional second argument *mu* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is - automatically calculated. + If the optional second argument *mu* is given, it is typically the mean of + the *data*. It can also be used to compute the second moment around a + point that is not the mean. If it is missing or ``None`` (the default), + the arithmetic mean is automatically calculated. Use this function to calculate the variance from the entire population. To estimate the variance from a sample, the :func:`variance` function is usually @@ -398,10 +415,6 @@ However, for reading convenience, most of the examples show sorted sequences. >>> pvariance(data, mu) 1.25 - This function does not attempt to verify that you have passed the actual mean - as *mu*. Using arbitrary values for *mu* may lead to invalid or impossible - results. - Decimals and Fractions are supported: .. doctest:: @@ -420,11 +433,11 @@ However, for reading convenience, most of the examples show sorted sequences. σ². When called on a sample instead, this is the biased sample variance s², also known as variance with N degrees of freedom. - If you somehow know the true population mean μ, you may use this function - to calculate the variance of a sample, giving the known population mean as - the second argument. Provided the data points are representative - (e.g. independent and identically distributed), the result will be an - unbiased estimate of the population variance. + If you somehow know the true population mean μ, you may use this + function to calculate the variance of a sample, giving the known + population mean as the second argument. Provided the data points are a + random sample of the population, the result will be an unbiased estimate + of the population variance. .. function:: stdev(data, xbar=None) @@ -499,6 +512,59 @@ However, for reading convenience, most of the examples show sorted sequences. :func:`pvariance` function as the *mu* parameter to get the variance of a sample. +.. function:: quantiles(data, *, n=4, method='exclusive') + + Divide *data* into *n* continuous intervals with equal probability. + Returns a list of ``n - 1`` cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. Set + *n* to 100 for percentiles which gives the 99 cuts points that separate + *data* into 100 equal sized groups. Raises :exc:`StatisticsError` if *n* + is not least 1. + + The *data* can be any iterable containing sample data. For meaningful + results, the number of data points in *data* should be larger than *n*. + Raises :exc:`StatisticsError` if there are not at least two data points. + + The cut points are linearly interpolated from the + two nearest data points. For example, if a cut point falls one-third + of the distance between two sample values, ``100`` and ``112``, the + cut-point will evaluate to ``104``. + + The *method* for computing quantiles can be varied depending on + whether the *data* includes or excludes the lowest and + highest possible values from the population. + + The default *method* is "exclusive" and is used for data sampled from + a population that can have more extreme values than found in the + samples. The portion of the population falling below the *i-th* of + *m* sorted data points is computed as ``i / (m + 1)``. Given nine + sample values, the method sorts them and assigns the following + percentiles: 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%. + + Setting the *method* to "inclusive" is used for describing population + data or for samples that are known to include the most extreme values + from the population. The minimum value in *data* is treated as the 0th + percentile and the maximum value is treated as the 100th percentile. + The portion of the population falling below the *i-th* of *m* sorted + data points is computed as ``(i - 1) / (m - 1)``. Given 11 sample + values, the method sorts them and assigns the following percentiles: + 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%. + + .. doctest:: + + # Decile cut points for empirically sampled data + >>> data = [105, 129, 87, 86, 111, 111, 89, 81, 108, 92, 110, + ... 100, 75, 105, 103, 109, 76, 119, 99, 91, 103, 129, + ... 106, 101, 84, 111, 74, 87, 86, 103, 103, 106, 86, + ... 111, 75, 87, 102, 121, 111, 88, 89, 101, 106, 95, + ... 103, 107, 101, 81, 109, 104] + >>> [round(q, 1) for q in quantiles(data, n=10)] + [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0] + + .. versionadded:: 3.8 + + Exceptions ---------- @@ -515,7 +581,7 @@ A single exception is defined: :class:`NormalDist` is a tool for creating and manipulating normal distributions of a `random variable `_. It is a -composite class that treats the mean and standard deviation of data +class that treats the mean and standard deviation of data measurements as a single entity. Normal distributions arise from the `Central Limit Theorem @@ -537,6 +603,18 @@ of applications in statistics. `_ of a normal distribution. + .. attribute:: median + + A read-only property for the `median + `_ of a normal + distribution. + + .. attribute:: mode + + A read-only property for the `mode + `_ of a normal + distribution. + .. attribute:: stdev A read-only property for the `standard deviation @@ -551,15 +629,16 @@ of applications in statistics. .. classmethod:: NormalDist.from_samples(data) - Makes a normal distribution instance computed from sample data. The - *data* can be any :term:`iterable` and should consist of values that - can be converted to type :class:`float`. + Makes a normal distribution instance with *mu* and *sigma* parameters + estimated from the *data* using :func:`fmean` and :func:`stdev`. - If *data* does not contain at least two elements, raises - :exc:`StatisticsError` because it takes at least one point to estimate - a central value and at least two points to estimate dispersion. + The *data* can be any :term:`iterable` and should consist of values + that can be converted to type :class:`float`. If *data* does not + contain at least two elements, raises :exc:`StatisticsError` because it + takes at least one point to estimate a central value and at least two + points to estimate dispersion. - .. method:: NormalDist.samples(n, seed=None) + .. method:: NormalDist.samples(n, *, seed=None) Generates *n* random samples for a given mean and standard deviation. Returns a :class:`list` of :class:`float` values. @@ -571,10 +650,10 @@ of applications in statistics. .. method:: NormalDist.pdf(x) Using a `probability density function (pdf) - `_, - compute the relative likelihood that a random variable *X* will be near - the given value *x*. Mathematically, it is the ratio ``P(x <= X < - x+dx) / dx``. + `_, compute + the relative likelihood that a random variable *X* will be near the + given value *x*. Mathematically, it is the limit of the ratio ``P(x <= + X < x+dx) / dx`` as *dx* approaches zero. The relative likelihood is computed as the probability of a sample occurring in a narrow range divided by the width of the range (hence @@ -602,13 +681,31 @@ of applications in statistics. .. method:: NormalDist.overlap(other) - Compute the `overlapping coefficient (OVL) - `_ - between two normal distributions, giving a measure of agreement. + Measures the agreement between two normal probability distributions. Returns a value between 0.0 and 1.0 giving `the overlapping area for - two probability density functions + the two probability density functions `_. + .. method:: NormalDist.quantiles(n=4) + + Divide the normal distribution into *n* continuous intervals with + equal probability. Returns a list of (n - 1) cut points separating + the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate the normal distribution into 100 equal sized groups. + + .. method:: NormalDist.zscore(x) + + Compute the + `Standard Score `_ + describing *x* in terms of the number of standard deviations + above or below the mean of the normal distribution: + ``(x - mean) / stdev``. + + .. versionadded:: 3.9 + Instances of :class:`NormalDist` support addition, subtraction, multiplication and division by a constant. These operations are used for translation and scaling. For example: @@ -647,10 +744,10 @@ of applications in statistics. :class:`NormalDist` readily solves classic probability problems. For example, given `historical data for SAT exams -`_ showing that scores -are normally distributed with a mean of 1060 and a standard deviation of 192, -determine the percentage of students with scores between 1100 and 1200, after -rounding to the nearest whole number: +`_ showing +that scores are normally distributed with a mean of 1060 and a standard +deviation of 195, determine the percentage of students with test scores +between 1100 and 1200, after rounding to the nearest whole number: .. doctest:: @@ -664,21 +761,11 @@ Find the `quartiles `_ and `deciles .. doctest:: - >>> [round(sat.inv_cdf(p)) for p in (0.25, 0.50, 0.75)] + >>> list(map(round, sat.quantiles())) [928, 1060, 1192] - >>> [round(sat.inv_cdf(p / 10)) for p in range(1, 10)] + >>> list(map(round, sat.quantiles(n=10))) [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] -What percentage of men and women will have the same height in `two normally -distributed populations with known means and standard deviations -`_? - - >>> men = NormalDist(70, 4) - >>> women = NormalDist(65, 3.5) - >>> ovl = men.overlap(women) - >>> round(ovl * 100.0, 1) - 50.3 - To estimate the distribution for a model than isn't easy to solve analytically, :class:`NormalDist` can generate input samples for a `Monte Carlo simulation `_: @@ -689,11 +776,47 @@ Carlo simulation `_: ... return (3*x + 7*x*y - 5*y) / (11 * z) ... >>> n = 100_000 - >>> X = NormalDist(10, 2.5).samples(n) - >>> Y = NormalDist(15, 1.75).samples(n) - >>> Z = NormalDist(5, 1.25).samples(n) - >>> NormalDist.from_samples(map(model, X, Y, Z)) # doctest: +SKIP - NormalDist(mu=19.640137307085507, sigma=47.03273142191088) + >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728) + >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471) + >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453) + >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP + [1.4591308524824727, 1.8035946855390597, 2.175091447274739] + +Normal distributions can be used to approximate `Binomial +distributions `_ +when the sample size is large and when the probability of a successful +trial is near 50%. + +For example, an open source conference has 750 attendees and two rooms with a +500 person capacity. There is a talk about Python and another about Ruby. +In previous conferences, 65% of the attendees preferred to listen to Python +talks. Assuming the population preferences haven't changed, what is the +probability that the Python room will stay within its capacity limits? + +.. doctest:: + + >>> n = 750 # Sample size + >>> p = 0.65 # Preference for Python + >>> q = 1.0 - p # Preference for Ruby + >>> k = 500 # Room capacity + + >>> # Approximation using the cumulative normal distribution + >>> from math import sqrt + >>> round(NormalDist(mu=n*p, sigma=sqrt(n*p*q)).cdf(k + 0.5), 4) + 0.8402 + + >>> # Solution using the cumulative binomial distribution + >>> from math import comb, fsum + >>> round(fsum(comb(n, r) * p**r * q**(n-r) for r in range(k+1)), 4) + 0.8402 + + >>> # Approximation using a simulation + >>> from random import seed, choices + >>> seed(8675309) + >>> def trial(): + ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') + >>> mean(trial() <= k for i in range(10_000)) + 0.8398 Normal distributions commonly arise in machine learning problems. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index aeecdbb24a57b1..2eee22c79af769 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -261,13 +261,14 @@ and imaginary parts. Python fully supports mixed arithmetic: when a binary arithmetic operator has operands of different numeric types, the operand with the "narrower" type is widened to that of the other, where integer is narrower than floating point, -which is narrower than complex. Comparisons between numbers of mixed type use -the same rule. [2]_ The constructors :func:`int`, :func:`float`, and +which is narrower than complex. A comparison between numbers of different types +behaves as though the exact values of those numbers were being compared. [2]_ + +The constructors :func:`int`, :func:`float`, and :func:`complex` can be used to produce numbers of a specific type. -All numeric types (except complex) support the following operations, sorted by -ascending priority (all numeric operations have a higher priority than -comparison operations): +All numeric types (except complex) support the following operations (for priorities of +the operations, see :ref:`operator-summary`): +---------------------+---------------------------------+---------+--------------------+ | Operation | Result | Notes | Full documentation | @@ -351,7 +352,7 @@ Notes: The numeric literals accepted include the digits ``0`` to ``9`` or any Unicode equivalent (code points with the ``Nd`` property). - See http://www.unicode.org/Public/12.0.0/ucd/extracted/DerivedNumericType.txt + See https://www.unicode.org/Public/13.0.0/ucd/extracted/DerivedNumericType.txt for a complete list of code points with the ``Nd`` property. @@ -433,12 +434,10 @@ Notes: Negative shift counts are illegal and cause a :exc:`ValueError` to be raised. (2) - A left shift by *n* bits is equivalent to multiplication by ``pow(2, n)`` - without overflow check. + A left shift by *n* bits is equivalent to multiplication by ``pow(2, n)``. (3) - A right shift by *n* bits is equivalent to division by ``pow(2, n)`` without - overflow check. + A right shift by *n* bits is equivalent to floor division by ``pow(2, n)``. (4) Performing these calculations with at least one extra sign extension bit in @@ -479,6 +478,27 @@ class`. In addition, it provides a few more methods: .. versionadded:: 3.1 +.. method:: int.bit_count() + + Return the number of ones in the binary representation of the absolute + value of the integer. This is also known as the population count. + Example:: + + >>> n = 19 + >>> bin(n) + '0b10011' + >>> n.bit_count() + 3 + >>> (-n).bit_count() + 3 + + Equivalent to:: + + def bit_count(self): + return bin(self).count("1") + + .. versionadded:: 3.10 + .. method:: int.to_bytes(length, byteorder, \*, signed=False) Return an array of bytes representing an integer. @@ -1114,7 +1134,7 @@ Notes: item is removed and returned. (3) - ``remove`` raises :exc:`ValueError` when *x* is not found in *s*. + :meth:`remove` raises :exc:`ValueError` when *x* is not found in *s*. (4) The :meth:`reverse` method modifies the sequence in place for economy of @@ -1124,7 +1144,9 @@ Notes: (5) :meth:`clear` and :meth:`!copy` are included for consistency with the interfaces of mutable containers that don't support slicing operations - (such as :class:`dict` and :class:`set`) + (such as :class:`dict` and :class:`set`). :meth:`!copy` is not part of the + :class:`collections.abc.MutableSequence` ABC, but most concrete + mutable sequence classes provide it. .. versionadded:: 3.3 :meth:`clear` and :meth:`!copy` methods. @@ -1382,7 +1404,7 @@ objects that compare equal might have different :attr:`~range.start`, .. seealso:: - * The `linspace recipe `_ + * The `linspace recipe `_ shows how to implement a lazy version of range suitable for floating point applications. @@ -1557,9 +1579,16 @@ expression support in the :mod:`re` module). :func:`codecs.register_error`, see section :ref:`error-handlers`. For a list of possible encodings, see section :ref:`standard-encodings`. + By default, the *errors* argument is not checked for best performances, but + only used at the first encoding error. Enable the :ref:`Python Development + Mode `, or use a debug build to check *errors*. + .. versionchanged:: 3.1 Support for keyword arguments added. + .. versionchanged:: 3.9 + The *errors* is now checked in development mode and in debug mode. + .. method:: str.endswith(suffix[, start[, end]]) @@ -1661,16 +1690,16 @@ expression support in the :mod:`re` module). .. method:: str.isalnum() - Return true if all characters in the string are alphanumeric and there is at - least one character, false otherwise. A character ``c`` is alphanumeric if one + Return ``True`` if all characters in the string are alphanumeric and there is at + least one character, ``False`` otherwise. A character ``c`` is alphanumeric if one of the following returns ``True``: ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or ``c.isnumeric()``. .. method:: str.isalpha() - Return true if all characters in the string are alphabetic and there is at least - one character, false otherwise. Alphabetic characters are those characters defined + Return ``True`` if all characters in the string are alphabetic and there is at least + one character, ``False`` otherwise. Alphabetic characters are those characters defined in the Unicode character database as "Letter", i.e., those with general category property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different from the "Alphabetic" property defined in the Unicode Standard. @@ -1678,8 +1707,8 @@ expression support in the :mod:`re` module). .. method:: str.isascii() - Return true if the string is empty or all characters in the string are ASCII, - false otherwise. + Return ``True`` if the string is empty or all characters in the string are ASCII, + ``False`` otherwise. ASCII characters have code points in the range U+0000-U+007F. .. versionadded:: 3.7 @@ -1687,8 +1716,8 @@ expression support in the :mod:`re` module). .. method:: str.isdecimal() - Return true if all characters in the string are decimal - characters and there is at least one character, false + Return ``True`` if all characters in the string are decimal + characters and there is at least one character, ``False`` otherwise. Decimal characters are those that can be used to form numbers in base 10, e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a decimal character is a character in the Unicode @@ -1697,8 +1726,8 @@ expression support in the :mod:`re` module). .. method:: str.isdigit() - Return true if all characters in the string are digits and there is at least one - character, false otherwise. Digits include decimal characters and digits that need + Return ``True`` if all characters in the string are digits and there is at least one + character, ``False`` otherwise. Digits include decimal characters and digits that need special handling, such as the compatibility superscript digits. This covers digits which cannot be used to form numbers in base 10, like the Kharosthi numbers. Formally, a digit is a character that has the @@ -1707,7 +1736,7 @@ expression support in the :mod:`re` module). .. method:: str.isidentifier() - Return true if the string is a valid identifier according to the language + Return ``True`` if the string is a valid identifier according to the language definition, section :ref:`identifiers`. Call :func:`keyword.iskeyword` to test whether string ``s`` is a reserved @@ -1726,14 +1755,14 @@ expression support in the :mod:`re` module). .. method:: str.islower() - Return true if all cased characters [4]_ in the string are lowercase and - there is at least one cased character, false otherwise. + Return ``True`` if all cased characters [4]_ in the string are lowercase and + there is at least one cased character, ``False`` otherwise. .. method:: str.isnumeric() - Return true if all characters in the string are numeric - characters, and there is at least one character, false + Return ``True`` if all characters in the string are numeric + characters, and there is at least one character, ``False`` otherwise. Numeric characters include digit characters, and all characters that have the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION ONE FIFTH. Formally, numeric characters are those with the property @@ -1742,8 +1771,8 @@ expression support in the :mod:`re` module). .. method:: str.isprintable() - Return true if all characters in the string are printable or the string is - empty, false otherwise. Nonprintable characters are those characters defined + Return ``True`` if all characters in the string are printable or the string is + empty, ``False`` otherwise. Nonprintable characters are those characters defined in the Unicode character database as "Other" or "Separator", excepting the ASCII space (0x20) which is considered printable. (Note that printable characters in this context are those which should not be escaped when @@ -1753,22 +1782,36 @@ expression support in the :mod:`re` module). .. method:: str.isspace() - Return true if there are only whitespace characters in the string and there is - at least one character, false otherwise. Whitespace characters are those - characters defined in the Unicode character database as "Other" or "Separator" - and those with bidirectional property being one of "WS", "B", or "S". + Return ``True`` if there are only whitespace characters in the string and there is + at least one character, ``False`` otherwise. + + A character is *whitespace* if in the Unicode character database + (see :mod:`unicodedata`), either its general category is ``Zs`` + ("Separator, space"), or its bidirectional class is one of ``WS``, + ``B``, or ``S``. + .. method:: str.istitle() - Return true if the string is a titlecased string and there is at least one + Return ``True`` if the string is a titlecased string and there is at least one character, for example uppercase characters may only follow uncased characters - and lowercase characters only cased ones. Return false otherwise. + and lowercase characters only cased ones. Return ``False`` otherwise. .. method:: str.isupper() - Return true if all cased characters [4]_ in the string are uppercase and - there is at least one cased character, false otherwise. + Return ``True`` if all cased characters [4]_ in the string are uppercase and + there is at least one cased character, ``False`` otherwise. + + >>> 'BANANA'.isupper() + True + >>> 'banana'.isupper() + False + >>> 'baNana'.isupper() + False + >>> ' '.isupper() + False + .. method:: str.join(iterable) @@ -1807,6 +1850,14 @@ expression support in the :mod:`re` module). >>> 'www.example.com'.lstrip('cmowz.') 'example.com' + See :meth:`str.removeprefix` for a method that will remove a single prefix + string rather than all of a set of characters. For example:: + + >>> 'Arthur: three!'.lstrip('Arthur: ') + 'ee!' + >>> 'Arthur: three!'.removeprefix('Arthur: ') + 'three!' + .. staticmethod:: str.maketrans(x[, y[, z]]) @@ -1831,6 +1882,34 @@ expression support in the :mod:`re` module). the string itself, followed by two empty strings. +.. method:: str.removeprefix(prefix, /) + + If the string starts with the *prefix* string, return + ``string[len(prefix):]``. Otherwise, return a copy of the original + string:: + + >>> 'TestHook'.removeprefix('Test') + 'Hook' + >>> 'BaseTestCase'.removeprefix('Test') + 'BaseTestCase' + + .. versionadded:: 3.9 + + +.. method:: str.removesuffix(suffix, /) + + If the string ends with the *suffix* string and that *suffix* is not empty, + return ``string[:-len(suffix)]``. Otherwise, return a copy of the + original string:: + + >>> 'MiscTests'.removesuffix('Tests') + 'Misc' + >>> 'TmpDirMixin'.removesuffix('Tests') + 'TmpDirMixin' + + .. versionadded:: 3.9 + + .. method:: str.replace(old, new[, count]) Return a copy of the string with all occurrences of substring *old* replaced by @@ -1887,6 +1966,13 @@ expression support in the :mod:`re` module). >>> 'mississippi'.rstrip('ipz') 'mississ' + See :meth:`str.removesuffix` for a method that will remove a single suffix + string rather than all of a set of characters. For example:: + + >>> 'Monty Python'.rstrip(' Python') + 'M' + >>> 'Monty Python'.removesuffix(' Python') + 'Monty' .. method:: str.split(sep=None, maxsplit=-1) @@ -2394,7 +2480,7 @@ data and are closely related to string objects in a variety of other ways. A reverse conversion function exists to transform a bytes object into its hexadecimal representation. - .. method:: hex() + .. method:: hex([sep[, bytes_per_sep]]) Return a string object containing two hexadecimal digits for each byte in the instance. @@ -2402,8 +2488,26 @@ data and are closely related to string objects in a variety of other ways. >>> b'\xf0\xf1\xf2'.hex() 'f0f1f2' + If you want to make the hex string easier to read, you can specify a + single character separator *sep* parameter to include in the output. + By default between each byte. A second optional *bytes_per_sep* + parameter controls the spacing. Positive values calculate the + separator position from the right, negative values from the left. + + >>> value = b'\xf0\xf1\xf2' + >>> value.hex('-') + 'f0-f1-f2' + >>> value.hex('_', 2) + 'f0_f1f2' + >>> b'UUDDLRLRAB'.hex(' ', -4) + '55554444 4c524c52 4142' + .. versionadded:: 3.5 + .. versionchanged:: 3.8 + :meth:`bytes.hex` now supports optional *sep* and *bytes_per_sep* + parameters to insert separators between bytes in the hex output. + Since bytes objects are sequences of integers (akin to a tuple), for a bytes object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes object of length 1. (This contrasts with text strings, where both indexing @@ -2470,7 +2574,7 @@ objects. A reverse conversion function exists to transform a bytearray object into its hexadecimal representation. - .. method:: hex() + .. method:: hex([sep[, bytes_per_sep]]) Return a string object containing two hexadecimal digits for each byte in the instance. @@ -2480,6 +2584,11 @@ objects. .. versionadded:: 3.5 + .. versionchanged:: 3.8 + Similar to :meth:`bytes.hex`, :meth:`bytearray.hex` now supports + optional *sep* and *bytes_per_sep* parameters to insert separators + between bytes in the hex output. + Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytearray object of length 1. (This contrasts with text strings, where @@ -2544,6 +2653,50 @@ arbitrary binary data. Also accept an integer in the range 0 to 255 as the subsequence. +.. method:: bytes.removeprefix(prefix, /) + bytearray.removeprefix(prefix, /) + + If the binary data starts with the *prefix* string, return + ``bytes[len(prefix):]``. Otherwise, return a copy of the original + binary data:: + + >>> b'TestHook'.removeprefix(b'Test') + b'Hook' + >>> b'BaseTestCase'.removeprefix(b'Test') + b'BaseTestCase' + + The *prefix* may be any :term:`bytes-like object`. + + .. note:: + + The bytearray version of this method does *not* operate in place - + it always produces a new object, even if no changes were made. + + .. versionadded:: 3.9 + + +.. method:: bytes.removesuffix(suffix, /) + bytearray.removesuffix(suffix, /) + + If the binary data ends with the *suffix* string and that *suffix* is + not empty, return ``bytes[:-len(suffix)]``. Otherwise, return a copy of + the original binary data:: + + >>> b'MiscTests'.removesuffix(b'Tests') + b'Misc' + >>> b'TmpDirMixin'.removesuffix(b'Tests') + b'TmpDirMixin' + + The *suffix* may be any :term:`bytes-like object`. + + .. note:: + + The bytearray version of this method does *not* operate in place - + it always produces a new object, even if no changes were made. + + .. versionadded:: 3.9 + + .. method:: bytes.decode(encoding="utf-8", errors="strict") bytearray.decode(encoding="utf-8", errors="strict") @@ -2555,6 +2708,10 @@ arbitrary binary data. :func:`codecs.register_error`, see section :ref:`error-handlers`. For a list of possible encodings, see section :ref:`standard-encodings`. + By default, the *errors* argument is not checked for best performances, but + only used at the first decoding error. Enable the :ref:`Python Development + Mode `, or use a debug build to check *errors*. + .. note:: Passing the *encoding* argument to :class:`str` allows decoding any @@ -2564,6 +2721,9 @@ arbitrary binary data. .. versionchanged:: 3.1 Added support for keyword arguments. + .. versionchanged:: 3.9 + The *errors* is now checked in development mode and in debug mode. + .. method:: bytes.endswith(suffix[, start[, end]]) bytearray.endswith(suffix[, start[, end]]) @@ -2716,8 +2876,8 @@ arbitrary binary data. The prefix(es) to search for may be any :term:`bytes-like object`. -.. method:: bytes.translate(table, delete=b'') - bytearray.translate(table, delete=b'') +.. method:: bytes.translate(table, /, delete=b'') + bytearray.translate(table, /, delete=b'') Return a copy of the bytes or bytearray object where all bytes occurring in the optional argument *delete* are removed, and the remaining bytes have @@ -2787,7 +2947,14 @@ produce new objects. b'example.com' The binary sequence of byte values to remove may be any - :term:`bytes-like object`. + :term:`bytes-like object`. See :meth:`~bytes.removeprefix` for a method + that will remove a single prefix string rather than all of a set of + characters. For example:: + + >>> b'Arthur: three!'.lstrip(b'Arthur: ') + b'ee!' + >>> b'Arthur: three!'.removeprefix(b'Arthur: ') + b'three!' .. note:: @@ -2836,7 +3003,14 @@ produce new objects. b'mississ' The binary sequence of byte values to remove may be any - :term:`bytes-like object`. + :term:`bytes-like object`. See :meth:`~bytes.removesuffix` for a method + that will remove a single suffix string rather than all of a set of + characters. For example:: + + >>> b'Monty Python'.rstrip(b' Python') + b'M' + >>> b'Monty Python'.removesuffix(b' Python') + b'Monty' .. note:: @@ -2963,8 +3137,8 @@ place, and instead produce new objects. .. method:: bytes.isalnum() bytearray.isalnum() - Return true if all bytes in the sequence are alphabetical ASCII characters - or ASCII decimal digits and the sequence is not empty, false otherwise. + Return ``True`` if all bytes in the sequence are alphabetical ASCII characters + or ASCII decimal digits and the sequence is not empty, ``False`` otherwise. Alphabetic ASCII characters are those byte values in the sequence ``b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'``. ASCII decimal digits are those byte values in the sequence ``b'0123456789'``. @@ -2980,8 +3154,8 @@ place, and instead produce new objects. .. method:: bytes.isalpha() bytearray.isalpha() - Return true if all bytes in the sequence are alphabetic ASCII characters - and the sequence is not empty, false otherwise. Alphabetic ASCII + Return ``True`` if all bytes in the sequence are alphabetic ASCII characters + and the sequence is not empty, ``False`` otherwise. Alphabetic ASCII characters are those byte values in the sequence ``b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'``. @@ -2996,8 +3170,8 @@ place, and instead produce new objects. .. method:: bytes.isascii() bytearray.isascii() - Return true if the sequence is empty or all bytes in the sequence are ASCII, - false otherwise. + Return ``True`` if the sequence is empty or all bytes in the sequence are ASCII, + ``False`` otherwise. ASCII bytes are in the range 0-0x7F. .. versionadded:: 3.7 @@ -3006,8 +3180,8 @@ place, and instead produce new objects. .. method:: bytes.isdigit() bytearray.isdigit() - Return true if all bytes in the sequence are ASCII decimal digits - and the sequence is not empty, false otherwise. ASCII decimal digits are + Return ``True`` if all bytes in the sequence are ASCII decimal digits + and the sequence is not empty, ``False`` otherwise. ASCII decimal digits are those byte values in the sequence ``b'0123456789'``. For example:: @@ -3021,8 +3195,8 @@ place, and instead produce new objects. .. method:: bytes.islower() bytearray.islower() - Return true if there is at least one lowercase ASCII character - in the sequence and no uppercase ASCII characters, false otherwise. + Return ``True`` if there is at least one lowercase ASCII character + in the sequence and no uppercase ASCII characters, ``False`` otherwise. For example:: @@ -3039,8 +3213,8 @@ place, and instead produce new objects. .. method:: bytes.isspace() bytearray.isspace() - Return true if all bytes in the sequence are ASCII whitespace and the - sequence is not empty, false otherwise. ASCII whitespace characters are + Return ``True`` if all bytes in the sequence are ASCII whitespace and the + sequence is not empty, ``False`` otherwise. ASCII whitespace characters are those byte values in the sequence ``b' \t\n\r\x0b\f'`` (space, tab, newline, carriage return, vertical tab, form feed). @@ -3048,8 +3222,8 @@ place, and instead produce new objects. .. method:: bytes.istitle() bytearray.istitle() - Return true if the sequence is ASCII titlecase and the sequence is not - empty, false otherwise. See :meth:`bytes.title` for more details on the + Return ``True`` if the sequence is ASCII titlecase and the sequence is not + empty, ``False`` otherwise. See :meth:`bytes.title` for more details on the definition of "titlecase". For example:: @@ -3063,8 +3237,8 @@ place, and instead produce new objects. .. method:: bytes.isupper() bytearray.isupper() - Return true if there is at least one uppercase alphabetic ASCII character - in the sequence and no lowercase ASCII characters, false otherwise. + Return ``True`` if there is at least one uppercase alphabetic ASCII character + in the sequence and no lowercase ASCII characters, ``False`` otherwise. For example:: @@ -3620,13 +3794,13 @@ copying. :mod:`struct` module syntax. .. versionadded:: 3.8 - *Order* can be {'C', 'F', 'A'}. When *order* is 'C' or 'F', the data + *order* can be {'C', 'F', 'A'}. When *order* is 'C' or 'F', the data of the original array is converted to C or Fortran order. For contiguous views, 'A' returns an exact copy of the physical memory. In particular, in-memory Fortran order is preserved. For non-contiguous views, the data is converted to C first. *order=None* is the same as *order='C'*. - .. method:: hex() + .. method:: hex([sep[, bytes_per_sep]]) Return a string object containing two hexadecimal digits for each byte in the buffer. :: @@ -3637,6 +3811,11 @@ copying. .. versionadded:: 3.5 + .. versionchanged:: 3.8 + Similar to :meth:`bytes.hex`, :meth:`memoryview.hex` now supports + optional *sep* and *bytes_per_sep* parameters to insert separators + between bytes in the hex output. + .. method:: tolist() Return the data in the buffer as a list of elements. :: @@ -3781,7 +3960,7 @@ copying. >>> z.nbytes 48 - Cast 1D/unsigned char to 2D/unsigned long:: + Cast 1D/unsigned long to 2D/unsigned long:: >>> buf = struct.pack("L"*6, *list(range(6))) >>> x = memoryview(buf) @@ -4027,7 +4206,7 @@ The constructors for both classes work the same: Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, and :meth:`symmetric_difference`, :meth:`issubset`, and + :meth:`difference`, :meth:`symmetric_difference`, :meth:`issubset`, and :meth:`issuperset` methods will accept any iterable as an argument. In contrast, their operator based counterparts require their arguments to be sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` @@ -4176,7 +4355,8 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3])) >>> d = dict([('two', 2), ('one', 1), ('three', 3)]) >>> e = dict({'three': 3, 'one': 1, 'two': 2}) - >>> a == b == c == d == e + >>> f = dict({'one': 1, 'three': 3}, two=2) + >>> a == b == c == d == e == f True Providing keyword arguments as in the first example only works for keys that @@ -4186,6 +4366,10 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: These are the operations that dictionaries support (and therefore, custom mapping types should support too): + .. describe:: list(d) + + Return a list of all the keys used in the dictionary *d*. + .. describe:: len(d) Return the number of items in the dictionary *d*. @@ -4254,7 +4438,10 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: Create a new dictionary with keys from *iterable* and values set to *value*. :meth:`fromkeys` is a class method that returns a new dictionary. *value* - defaults to ``None``. + defaults to ``None``. All of the values refer to just a single instance, + so it generally doesn't make sense for *value* to be a mutable object + such as an empty list. To get distinct values, use a :ref:`dict + comprehension ` instead. .. method:: get(key[, default]) @@ -4296,6 +4483,8 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: Return a reverse iterator over the keys of the dictionary. This is a shortcut for ``reversed(d.keys())``. + .. versionadded:: 3.8 + .. method:: setdefault(key[, default]) If *key* is in the dictionary, return its value. If not, insert *key* @@ -4317,8 +4506,32 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: Return a new view of the dictionary's values. See the :ref:`documentation of view objects `. + An equality comparison between one ``dict.values()`` view and another + will always return ``False``. This also applies when comparing + ``dict.values()`` to itself:: + + >>> d = {'a': 1} + >>> d.values() == d.values() + False + + .. describe:: d | other + + Create a new dictionary with the merged keys and values of *d* and + *other*, which must both be dictionaries. The values of *other* take + priority when *d* and *other* share keys. + + .. versionadded:: 3.9 + + .. describe:: d |= other + + Update the dictionary *d* with keys and values from *other*, which may be + either a :term:`mapping` or an :term:`iterable` of key/value pairs. The + values of *other* take priority when *d* and *other* share keys. + + .. versionadded:: 3.9 + Dictionaries compare equal if and only if they have the same ``(key, - value)`` pairs. Order comparisons ('<', '<=', '>=', '>') raise + value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise :exc:`TypeError`. Dictionaries preserve insertion order. Note that updating a key does not @@ -4410,6 +4623,12 @@ support membership tests: .. versionchanged:: 3.8 Dictionary views are now reversible. +.. describe:: dictview.mapping + + Return a :class:`types.MappingProxyType` that wraps the original + dictionary to which the view refers. + + .. versionadded:: 3.10 Keys views are set-like since their entries are unique and hashable. If all values are hashable, so that ``(key, value)`` pairs are unique and hashable, @@ -4449,6 +4668,12 @@ An example of dictionary view usage:: >>> keys ^ {'sausage', 'juice'} {'juice', 'sausage', 'bacon', 'spam'} + >>> # get back a read-only proxy for the original dictionary + >>> values.mapping + mappingproxy({'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500}) + >>> values.mapping['spam'] + 500 + .. _typecontextmanager: diff --git a/Doc/library/string.rst b/Doc/library/string.rst index c2f65224bc8daf..91f43e9353d915 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -88,7 +88,7 @@ implementation as the built-in :meth:`~str.format` method. The :class:`Formatter` class has the following public methods: - .. method:: format(format_string, *args, **kwargs) + .. method:: format(format_string, /, *args, **kwargs) The primary API method. It takes a format string and an arbitrary set of positional and keyword arguments. @@ -146,7 +146,7 @@ implementation as the built-in :meth:`~str.format` method. keyword arguments. For compound field names, these functions are only called for the first - component of the field name; Subsequent components are handled through + component of the field name; subsequent components are handled through normal attribute and indexing operations. So for example, the field expression '0.name' would cause @@ -205,7 +205,7 @@ literal text, it can be escaped by doubling: ``{{`` and ``}}``. The grammar for a replacement field is as follows: - .. productionlist:: sf + .. productionlist:: format-string replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}" field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")* arg_name: [`identifier` | `digit`+] @@ -302,13 +302,13 @@ specification is to be interpreted. Most built-in types implement the following options for format specifications, although some of the formatting options are only supported by the numeric types. -A general convention is that an empty format string (``""``) produces +A general convention is that an empty format specification produces the same result as if you had called :func:`str` on the value. A -non-empty format string typically modifies the result. +non-empty format specification typically modifies the result. The general form of a *standard format specifier* is: -.. productionlist:: sf +.. productionlist:: format-spec format_spec: [[`fill`]`align`][`sign`][#][0][`width`][`grouping_option`][.`precision`][`type`] fill: align: "<" | ">" | "=" | "^" @@ -415,8 +415,9 @@ error. .. versionchanged:: 3.6 Added the ``'_'`` option (see also :pep:`515`). -*width* is a decimal integer defining the minimum field width. If not -specified, then the field width will be determined by the content. +*width* is a decimal integer defining the minimum total field width, +including any prefixes, separators, and other formatting characters. +If not specified, then the field width will be determined by the content. When no explicit alignment is given, preceding the *width* field by a zero (``'0'``) character enables @@ -500,14 +501,16 @@ The available presentation types for floating point and decimal values are: | | | | | The precise rules are as follows: suppose that the | | | result formatted with presentation type ``'e'`` and | - | | precision ``p-1`` would have exponent ``exp``. Then | - | | if ``-4 <= exp < p``, the number is formatted | - | | with presentation type ``'f'`` and precision | + | | precision ``p-1`` would have exponent ``exp``. Then, | + | | if ``m <= exp < p``, where ``m`` is -4 for floats and -6 | + | | for :class:`Decimals `, the number is | + | | formatted with presentation type ``'f'`` and precision | | | ``p-1-exp``. Otherwise, the number is formatted | | | with presentation type ``'e'`` and precision ``p-1``. | | | In both cases insignificant trailing zeros are removed | | | from the significand, and the decimal point is also | - | | removed if there are no remaining digits following it. | + | | removed if there are no remaining digits following it, | + | | unless the ``'#'`` option is used. | | | | | | Positive and negative infinity, positive and negative | | | zero, and nans, are formatted as ``inf``, ``-inf``, | @@ -720,7 +723,7 @@ these rules. The methods of :class:`Template` are: The constructor takes a single argument which is the template string. - .. method:: substitute(mapping, **kwds) + .. method:: substitute(mapping={}, /, **kwds) Performs the template substitution, returning a new string. *mapping* is any dictionary-like object with keys that match the placeholders in the @@ -729,7 +732,7 @@ these rules. The methods of :class:`Template` are: and there are duplicates, the placeholders from *kwds* take precedence. - .. method:: safe_substitute(mapping, **kwds) + .. method:: safe_substitute(mapping={}, /, **kwds) Like :meth:`substitute`, except that if placeholders are missing from *mapping* and *kwds*, instead of raising a :exc:`KeyError` exception, the diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index 330032ba1c0ba6..5cfb533d802db4 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -34,7 +34,7 @@ itself was generated using the ``mkstringprep.py`` utility. As a result, these tables are exposed as functions, not as data structures. There are two kinds of tables in the RFC: sets and mappings. For a set, :mod:`stringprep` provides the "characteristic function", i.e. a function that -returns true if the parameter is part of the set. For mappings, it provides the +returns ``True`` if the parameter is part of the set. For mappings, it provides the mapping function: given the key, it returns the associated value. Below is a list of all functions available in the module. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 1a0fd73c6758c7..eccba20fb8fe7e 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -70,7 +70,7 @@ The module defines the following exception and functions: size required by the format, as reflected by :func:`calcsize`. -.. function:: unpack_from(format, buffer, offset=0) +.. function:: unpack_from(format, /, buffer, offset=0) Unpack from *buffer* starting at position *offset*, according to the format string *format*. The result is a tuple even if it contains exactly one @@ -159,8 +159,8 @@ the :ref:`format-characters` section. Note the difference between ``'@'`` and ``'='``: both use native byte order, but the size and alignment of the latter is standardized. -The form ``'!'`` is available for those poor souls who claim they can't remember -whether network byte order is big-endian or little-endian. +The form ``'!'`` represents the network byte order which is always big-endian +as defined in `IETF RFC 1700 `_. There is no way to indicate non-native byte order (force byte-swapping); use the appropriate choice of ``'<'`` or ``'>'``. @@ -197,44 +197,44 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``c`` | :c:type:`char` | bytes of length 1 | 1 | | +--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:type:`signed char` | integer | 1 | \(1),\(3) | +| ``b`` | :c:type:`signed char` | integer | 1 | \(1), \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:type:`unsigned char` | integer | 1 | \(3) | +| ``B`` | :c:type:`unsigned char` | integer | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ | ``?`` | :c:type:`_Bool` | bool | 1 | \(1) | +--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:type:`short` | integer | 2 | \(3) | +| ``h`` | :c:type:`short` | integer | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:type:`unsigned short` | integer | 2 | \(3) | +| ``H`` | :c:type:`unsigned short` | integer | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:type:`int` | integer | 4 | \(3) | +| ``i`` | :c:type:`int` | integer | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:type:`unsigned int` | integer | 4 | \(3) | +| ``I`` | :c:type:`unsigned int` | integer | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:type:`long` | integer | 4 | \(3) | +| ``l`` | :c:type:`long` | integer | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:type:`unsigned long` | integer | 4 | \(3) | +| ``L`` | :c:type:`unsigned long` | integer | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:type:`long long` | integer | 8 | \(2), \(3) | +| ``q`` | :c:type:`long long` | integer | 8 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:type:`unsigned long | integer | 8 | \(2), \(3) | +| ``Q`` | :c:type:`unsigned long | integer | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(4) | +| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(4) | +| ``N`` | :c:type:`size_t` | integer | | \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``e`` | \(7) | float | 2 | \(5) | +| ``e`` | \(6) | float | 2 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ -| ``f`` | :c:type:`float` | float | 4 | \(5) | +| ``f`` | :c:type:`float` | float | 4 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ -| ``d`` | :c:type:`double` | float | 8 | \(5) | +| ``d`` | :c:type:`double` | float | 8 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ | ``s`` | :c:type:`char[]` | bytes | | | +--------+--------------------------+--------------------+----------------+------------+ | ``p`` | :c:type:`char[]` | bytes | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:type:`void \*` | integer | | \(6) | +| ``P`` | :c:type:`void \*` | integer | | \(5) | +--------+--------------------------+--------------------+----------------+------------+ .. versionchanged:: 3.3 @@ -254,38 +254,33 @@ Notes: standard mode, it is always represented by one byte. (2) - The ``'q'`` and ``'Q'`` conversion codes are available in native mode only if - the platform C compiler supports C :c:type:`long long`, or, on Windows, - :c:type:`__int64`. They are always available in standard modes. - -(3) When attempting to pack a non-integer using any of the integer conversion codes, if the non-integer has a :meth:`__index__` method then that method is called to convert the argument to an integer before packing. .. versionchanged:: 3.2 - Use of the :meth:`__index__` method for non-integers is new in 3.2. + Added use of the :meth:`__index__` method for non-integers. -(4) +(3) The ``'n'`` and ``'N'`` conversion codes are only available for the native size (selected as the default or with the ``'@'`` byte order character). For the standard size, you can use whichever of the other integer formats fits your application. -(5) +(4) For the ``'f'``, ``'d'`` and ``'e'`` conversion codes, the packed representation uses the IEEE 754 binary32, binary64 or binary16 format (for ``'f'``, ``'d'`` or ``'e'`` respectively), regardless of the floating-point format used by the platform. -(6) +(5) The ``'P'`` format character is only available for the native byte ordering (selected as the default or with the ``'@'`` byte order character). The byte order character ``'='`` chooses to use little- or big-endian ordering based on the host system. The struct module does not interpret this as native ordering, so the ``'P'`` format is not available. -(7) +(6) The IEEE 754 binary16 "half precision" type was introduced in the 2008 revision of the `IEEE 754 standard `_. It has a sign bit, a 5-bit exponent and 11-bit precision (with 10 bits explicitly stored), @@ -317,7 +312,7 @@ When packing a value ``x`` using one of the integer formats (``'b'``, then :exc:`struct.error` is raised. .. versionchanged:: 3.1 - In 3.0, some of the integer formats wrapped out-of-range values and + Previously, some of the integer formats wrapped out-of-range values and raised :exc:`DeprecationWarning` instead of :exc:`struct.error`. The ``'p'`` format character encodes a "Pascal string", meaning a short @@ -472,3 +467,5 @@ The :mod:`struct` module also defines the following type: .. _half precision format: https://en.wikipedia.org/wiki/Half-precision_floating-point_format .. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_floating_point#IEEE_754-2008 + +.. _IETF RFC 1700: https://tools.ietf.org/html/rfc1700 diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ca0813c7830ad0..e37cc980e97575 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -40,7 +40,7 @@ compatibility with older versions, see the :ref:`call-function-trio` section. .. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ capture_output=False, shell=False, cwd=None, timeout=None, \ check=False, encoding=None, errors=None, text=None, env=None, \ - universal_newlines=None) + universal_newlines=None, **other_popen_kwargs) Run the command described by *args*. Wait for command to complete, then return a :class:`CompletedProcess` instance. @@ -55,7 +55,9 @@ compatibility with older versions, see the :ref:`call-function-trio` section. If *capture_output* is true, stdout and stderr will be captured. When used, the internal :class:`Popen` object is automatically created with ``stdout=PIPE`` and ``stderr=PIPE``. The *stdout* and *stderr* arguments may - not be supplied at the same time as *capture_output*. + not be supplied at the same time as *capture_output*. If you wish to capture + and combine both streams into one, use ``stdout=PIPE`` and ``stderr=STDOUT`` + instead of *capture_output*. The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout expires, the child process will be killed and waited for. The @@ -337,7 +339,8 @@ functions. stderr=None, preexec_fn=None, close_fds=True, shell=False, \ cwd=None, env=None, universal_newlines=None, \ startupinfo=None, creationflags=0, restore_signals=True, \ - start_new_session=False, pass_fds=(), *, \ + start_new_session=False, pass_fds=(), \*, group=None, \ + extra_groups=None, user=None, umask=-1, \ encoding=None, errors=None, text=None) Execute a child program in a new process. On POSIX, the class uses @@ -345,21 +348,28 @@ functions. the class uses the Windows ``CreateProcess()`` function. The arguments to :class:`Popen` are as follows. - *args* should be a sequence of program arguments or else a single string. + *args* should be a sequence of program arguments or else a single string + or :term:`path-like object`. By default, the program to execute is the first item in *args* if *args* is a sequence. If *args* is a string, the interpretation is platform-dependent and described below. See the *shell* and *executable* arguments for additional differences from the default behavior. Unless otherwise stated, it is recommended to pass *args* as a sequence. + An example of passing some arguments to an external program + as a sequence is:: + + Popen(["/usr/bin/git", "commit", "-m", "Fixes a bug."]) + On POSIX, if *args* is a string, the string is interpreted as the name or path of the program to execute. However, this can only be done if not passing arguments to the program. .. note:: - :meth:`shlex.split` can be useful when determining the correct - tokenization for *args*, especially in complex cases:: + It may not be obvious how to break a shell command into a sequence of arguments, + especially in complex cases. :meth:`shlex.split` can illustrate how to + determine the correct tokenization for *args*:: >>> import shlex, subprocess >>> command_line = input() @@ -379,6 +389,15 @@ functions. manner described in :ref:`converting-argument-sequence`. This is because the underlying ``CreateProcess()`` operates on strings. + .. versionchanged:: 3.6 + *args* parameter accepts a :term:`path-like object` if *shell* is + ``False`` and a sequence containing path-like objects on POSIX. + + .. versionchanged:: 3.8 + *args* parameter accepts a :term:`path-like object` if *shell* is + ``False`` and a sequence containing bytes and path-like objects + on Windows. + The *shell* argument (which defaults to ``False``) specifies whether to use the shell as the program to execute. If *shell* is ``True``, it is recommended to pass *args* as a string rather than as a sequence. @@ -434,6 +453,13 @@ functions. :program:`ps`. If ``shell=True``, on POSIX the *executable* argument specifies a replacement shell for the default :file:`/bin/sh`. + .. versionchanged:: 3.6 + *executable* parameter accepts a :term:`path-like object` on POSIX. + + .. versionchanged:: 3.8 + *executable* parameter accepts a bytes and :term:`path-like object` + on Windows. + *stdin*, *stdout* and *stderr* specify the executed program's standard input, standard output and standard error file handles, respectively. Valid values are :data:`PIPE`, :data:`DEVNULL`, an existing file descriptor (a positive @@ -464,6 +490,13 @@ functions. The *start_new_session* parameter can take the place of a previously common use of *preexec_fn* to call os.setsid() in the child. + .. versionchanged:: 3.8 + + The *preexec_fn* parameter is no longer supported in subinterpreters. + The use of the parameter in a subinterpreter raises + :exc:`RuntimeError`. The new restriction may affect applications that + are deployed in mod_wsgi, uWSGI, and other embedded environments. + If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and :const:`2` will be closed before the child process is executed. Otherwise when *close_fds* is false, file descriptors obey their inheritable flag @@ -490,13 +523,19 @@ functions. The *pass_fds* parameter was added. If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. *cwd* can be a :class:`str` and + *cwd* before executing the child. *cwd* can be a string, bytes or :term:`path-like ` object. In particular, the function looks for *executable* (or for the first item in *args*) relative to *cwd* if the executable path is a relative path. .. versionchanged:: 3.6 - *cwd* parameter accepts a :term:`path-like object`. + *cwd* parameter accepts a :term:`path-like object` on POSIX. + + .. versionchanged:: 3.7 + *cwd* parameter accepts a :term:`path-like object` on Windows. + + .. versionchanged:: 3.8 + *cwd* parameter accepts a bytes object on Windows. If *restore_signals* is true (the default) all signals that Python has set to SIG_IGN are restored to SIG_DFL in the child process before the exec. @@ -512,6 +551,39 @@ functions. .. versionchanged:: 3.2 *start_new_session* was added. + If *group* is not ``None``, the setregid() system call will be made in the + child process prior to the execution of the subprocess. If the provided + value is a string, it will be looked up via :func:`grp.getgrnam()` and + the value in ``gr_gid`` will be used. If the value is an integer, it + will be passed verbatim. (POSIX only) + + .. availability:: POSIX + .. versionadded:: 3.9 + + If *extra_groups* is not ``None``, the setgroups() system call will be + made in the child process prior to the execution of the subprocess. + Strings provided in *extra_groups* will be looked up via + :func:`grp.getgrnam()` and the values in ``gr_gid`` will be used. + Integer values will be passed verbatim. (POSIX only) + + .. availability:: POSIX + .. versionadded:: 3.9 + + If *user* is not ``None``, the setreuid() system call will be made in the + child process prior to the execution of the subprocess. If the provided + value is a string, it will be looked up via :func:`pwd.getpwnam()` and + the value in ``pw_uid`` will be used. If the value is an integer, it will + be passed verbatim. (POSIX only) + + .. availability:: POSIX + .. versionadded:: 3.9 + + If *umask* is not negative, the umask() system call will be made in the + child process prior to the execution of the subprocess. + + .. availability:: POSIX + .. versionadded:: 3.9 + If *env* is not ``None``, it must be a mapping that defines the environment variables for the new process; these are used instead of the default behavior of inheriting the current process' environment. @@ -560,6 +632,13 @@ functions. with Popen(["ifconfig"], stdout=PIPE) as proc: log.write(proc.stdout.read()) + .. audit-event:: subprocess.Popen executable,args,cwd,env subprocess.Popen + + Popen and the other functions in this module that use it raise an + :ref:`auditing event ` ``subprocess.Popen`` with arguments + ``executable``, ``args``, ``cwd``, and ``env``. The value for ``args`` + may be a single string or a list of strings, depending on platform. + .. versionchanged:: 3.2 Added context manager support. @@ -567,6 +646,13 @@ functions. Popen destructor now emits a :exc:`ResourceWarning` warning if the child process is still running. + .. versionchanged:: 3.8 + Popen can use :func:`os.posix_spawn` in some cases for better + performance. On Windows Subsystem for Linux and QEMU User Emulation, + Popen constructor using :func:`os.posix_spawn` no longer raise an + exception on errors like missing program, but the child process fails + with a non-zero :attr:`~Popen.returncode`. + Exceptions ^^^^^^^^^^ @@ -652,10 +738,11 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.communicate(input=None, timeout=None) Interact with process: Send data to stdin. Read data from stdout and stderr, - until end-of-file is reached. Wait for process to terminate. The optional - *input* argument should be data to be sent to the child process, or - ``None``, if no data should be sent to the child. If streams were opened in - text mode, *input* must be a string. Otherwise, it must be bytes. + until end-of-file is reached. Wait for process to terminate and set the + :attr:`~Popen.returncode` attribute. The optional *input* argument should be + data to be sent to the child process, or ``None``, if no data should be sent + to the child. If streams were opened in text mode, *input* must be a string. + Otherwise, it must be bytes. :meth:`communicate` returns a tuple ``(stdout_data, stderr_data)``. The data will be strings if streams were opened in text mode; otherwise, @@ -694,6 +781,8 @@ Instances of the :class:`Popen` class have the following methods: Sends the signal *signal* to the child. + Do nothing if the process completed. + .. note:: On Windows, SIGTERM is an alias for :meth:`terminate`. CTRL_C_EVENT and @@ -703,14 +792,14 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.terminate() - Stop the child. On Posix OSs the method sends SIGTERM to the + Stop the child. On POSIX OSs the method sends SIGTERM to the child. On Windows the Win32 API function :c:func:`TerminateProcess` is called to stop the child. .. method:: Popen.kill() - Kills the child. On Posix OSs the function sends SIGKILL to the child. + Kills the child. On POSIX OSs the function sends SIGKILL to the child. On Windows :meth:`kill` is an alias for :meth:`terminate`. @@ -997,12 +1086,13 @@ Prior to Python 3.5, these three functions comprised the high level API to subprocess. You can now use :func:`run` in many cases, but lots of existing code calls these functions. -.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None) +.. function:: call(args, *, stdin=None, stdout=None, stderr=None, \ + shell=False, cwd=None, timeout=None, **other_popen_kwargs) Run the command described by *args*. Wait for command to complete, then return the :attr:`~Popen.returncode` attribute. - Code needing to capture stdout or stderr should use :func:`run` instead: + Code needing to capture stdout or stderr should use :func:`run` instead:: run(...).returncode @@ -1023,14 +1113,16 @@ calls these functions. .. versionchanged:: 3.3 *timeout* was added. -.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None) +.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, \ + shell=False, cwd=None, timeout=None, \ + **other_popen_kwargs) Run command with arguments. Wait for command to complete. If the return code was zero then return, otherwise raise :exc:`CalledProcessError`. The :exc:`CalledProcessError` object will have the return code in the :attr:`~CalledProcessError.returncode` attribute. - Code needing to capture stdout or stderr should use :func:`run` instead: + Code needing to capture stdout or stderr should use :func:`run` instead:: run(..., check=True) @@ -1054,7 +1146,8 @@ calls these functions. .. function:: check_output(args, *, stdin=None, stderr=None, shell=False, \ cwd=None, encoding=None, errors=None, \ - universal_newlines=None, timeout=None, text=None) + universal_newlines=None, timeout=None, text=None, \ + **other_popen_kwargs) Run command with arguments and return its output. @@ -1127,12 +1220,12 @@ In the following examples, we assume that the relevant functions have already been imported from the :mod:`subprocess` module. -Replacing /bin/sh shell backquote -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Replacing :program:`/bin/sh` shell command substitution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: bash - output=`mycmd myarg` + output=$(mycmd myarg) becomes:: @@ -1143,7 +1236,7 @@ Replacing shell pipeline .. code-block:: bash - output=`dmesg | grep hda` + output=$(dmesg | grep hda) becomes:: @@ -1152,15 +1245,15 @@ becomes:: p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. output = p2.communicate()[0] -The p1.stdout.close() call after starting the p2 is important in order for p1 -to receive a SIGPIPE if p2 exits before p1. +The ``p1.stdout.close()`` call after starting the p2 is important in order for +p1 to receive a SIGPIPE if p2 exits before p1. Alternatively, for trusted input, the shell's own pipeline support may still be used directly: .. code-block:: bash - output=`dmesg | grep hda` + output=$(dmesg | grep hda) becomes:: diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst index 2064fd7e20bfc2..aad6f93b6bff18 100644 --- a/Doc/library/sunau.rst +++ b/Doc/library/sunau.rst @@ -59,13 +59,6 @@ The :mod:`sunau` module defines the following functions: or ``'wb'`` returns an :class:`AU_write` object. -.. function:: openfp(file, mode) - - A synonym for :func:`.open`, maintained for backwards compatibility. - - .. deprecated-removed:: 3.7 3.9 - - The :mod:`sunau` module defines the following exception: .. exception:: Error diff --git a/Doc/library/symbol.rst b/Doc/library/symbol.rst deleted file mode 100644 index 44996936e2d28f..00000000000000 --- a/Doc/library/symbol.rst +++ /dev/null @@ -1,27 +0,0 @@ -:mod:`symbol` --- Constants used with Python parse trees -======================================================== - -.. module:: symbol - :synopsis: Constants representing internal nodes of the parse tree. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/symbol.py` - --------------- - -This module provides constants which represent the numeric values of internal -nodes of the parse tree. Unlike most Python constants, these use lower-case -names. Refer to the file :file:`Grammar/Grammar` in the Python distribution for -the definitions of the names in the context of the language grammar. The -specific numeric values which the names map to may change between Python -versions. - -This module also provides one additional data object: - - -.. data:: sym_name - - Dictionary mapping the numeric values of the constants defined in this module - back to name strings, allowing more human-readable representation of parse trees - to be generated. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 7c6ac4dccf8b76..c9521d649b893e 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -67,10 +67,6 @@ Examining Symbol Tables Return ``True`` if the block has nested namespaces within it. These can be obtained with :meth:`get_children`. - .. method:: has_exec() - - Return ``True`` if the block uses ``exec``. - .. method:: get_identifiers() Return a list of names of symbols in this table. @@ -160,6 +156,10 @@ Examining Symbol Tables Return ``True`` if the symbol is local to its block. + .. method:: is_annotated() + + Return ``True`` if the symbol is annotated. + .. method:: is_free() Return ``True`` if the symbol is referenced in its block, but not assigned diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 591972e9b783a0..aa417ede402286 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -16,9 +16,45 @@ always available. On POSIX systems where Python was built with the standard ``configure`` script, this contains the ABI flags as specified by :pep:`3149`. + .. versionchanged:: 3.8 + Default flags became an empty string (``m`` flag for pymalloc has been + removed). + .. versionadded:: 3.2 +.. function:: addaudithook(hook) + + Append the callable *hook* to the list of active auditing hooks for the + current interpreter. + + When an auditing event is raised through the :func:`sys.audit` function, each + hook will be called in the order it was added with the event name and the + tuple of arguments. Native hooks added by :c:func:`PySys_AddAuditHook` are + called first, followed by hooks added in the current interpreter. + + .. audit-event:: sys.addaudithook "" sys.addaudithook + + Raise an auditing event ``sys.addaudithook`` with no arguments. If any + existing hooks raise an exception derived from :class:`RuntimeError`, the + new hook will not be added and the exception suppressed. As a result, + callers cannot assume that their hook has been added unless they control + all existing hooks. + + .. versionadded:: 3.8 + + .. versionchanged:: 3.8.1 + + Exceptions derived from :class:`Exception` but not :class:`RuntimeError` + are no longer suppressed. + + .. impl-detail:: + + When tracing is enabled (see :func:`settrace`), Python hooks are only + traced if the callable has a ``__cantrace__`` member that is set to a + true value. Otherwise, trace functions will skip the hook. + + .. data:: argv The list of command line arguments passed to a Python script. ``argv[0]`` is the @@ -30,6 +66,8 @@ always available. To loop over the standard input, or the list of files given on the command line, see the :mod:`fileinput` module. + See also :data:`sys.orig_argv`. + .. note:: On Unix, command line arguments are passed by bytes from OS. Python decodes them with filesystem encoding and "surrogateescape" error handler. @@ -37,6 +75,33 @@ always available. ``[os.fsencode(arg) for arg in sys.argv]``. +.. _auditing: + +.. function:: audit(event, *args) + + .. index:: single: auditing + + Raise an auditing event with any active hooks. The event name is a string + identifying the event and its associated schema, which is the number and + types of arguments. The schema for a given event is considered public and + stable API and should not be modified between releases. + + This function will raise the first exception raised by any hook. In general, + these errors should not be handled and should terminate the process as + quickly as possible. + + Hooks are added using the :func:`sys.addaudithook` or + :c:func:`PySys_AddAuditHook` functions. + + The native equivalent of this function is :c:func:`PySys_Audit`. Using the + native function is preferred when possible. + + See the :ref:`audit events table ` for all events raised by + CPython. + + .. versionadded:: 3.8 + + .. data:: base_exec_prefix Set during Python startup, before ``site.py`` is run, to the same value as @@ -114,6 +179,8 @@ always available. This function should be used for internal and specialized purposes only. + .. audit-event:: sys._current_frames "" sys._current_frames + .. function:: breakpointhook() @@ -248,20 +315,38 @@ always available. before the program exits. The handling of such top-level exceptions can be customized by assigning another three-argument function to ``sys.excepthook``. + .. audit-event:: sys.excepthook hook,type,value,traceback sys.excepthook + + Raise an auditing event ``sys.excepthook`` with arguments ``hook``, + ``type``, ``value``, ``traceback`` when an uncaught exception occurs. + If no hook has been set, ``hook`` may be ``None``. If any hook raises + an exception derived from :class:`RuntimeError` the call to the hook will + be suppressed. Otherwise, the audit hook exception will be reported as + unraisable and ``sys.excepthook`` will be called. + + .. seealso:: + + The :func:`sys.unraisablehook` function handles unraisable exceptions + and the :func:`threading.excepthook` function handles exception raised + by :func:`threading.Thread.run`. + .. data:: __breakpointhook__ __displayhook__ __excepthook__ + __unraisablehook__ These objects contain the original values of ``breakpointhook``, - ``displayhook``, and ``excepthook`` at the start of the program. They are - saved so that ``breakpointhook``, ``displayhook`` and ``excepthook`` can be - restored in case they happen to get replaced with broken or alternative - objects. + ``displayhook``, ``excepthook``, and ``unraisablehook`` at the start of the + program. They are saved so that ``breakpointhook``, ``displayhook`` and + ``excepthook``, ``unraisablehook`` can be restored in case they happen to + get replaced with broken or alternative objects. .. versionadded:: 3.7 __breakpointhook__ + .. versionadded:: 3.8 + __unraisablehook__ .. function:: exc_info() @@ -281,7 +366,7 @@ always available. ``(type, value, traceback)``. Their meaning is: *type* gets the type of the exception being handled (a subclass of :exc:`BaseException`); *value* gets the exception instance (an instance of the exception type); *traceback* gets - a traceback object (see the Reference Manual) which encapsulates the call + a :ref:`traceback object ` which encapsulates the call stack at the point where the exception originally occurred. @@ -344,12 +429,12 @@ always available. .. data:: flags - The :term:`struct sequence` *flags* exposes the status of command line + The :term:`named tuple` *flags* exposes the status of command line flags. The attributes are read only. - ============================= ============================= + ============================= ================================================================ attribute flag - ============================= ============================= + ============================= ================================================================ :const:`debug` :option:`-d` :const:`inspect` :option:`-i` :const:`interactive` :option:`-i` @@ -363,9 +448,9 @@ always available. :const:`bytes_warning` :option:`-b` :const:`quiet` :option:`-q` :const:`hash_randomization` :option:`-R` - :const:`dev_mode` :option:`-X` ``dev`` - :const:`utf8_mode` :option:`-X` ``utf8`` - ============================= ============================= + :const:`dev_mode` :option:`-X dev <-X>` (:ref:`Python Development Mode `) + :const:`utf8_mode` :option:`-X utf8 <-X>` + ============================= ================================================================ .. versionchanged:: 3.2 Added ``quiet`` attribute for the new :option:`-q` flag. @@ -380,13 +465,14 @@ always available. Added ``isolated`` attribute for :option:`-I` ``isolated`` flag. .. versionchanged:: 3.7 - Added ``dev_mode`` attribute for the new :option:`-X` ``dev`` flag - and ``utf8_mode`` attribute for the new :option:`-X` ``utf8`` flag. + Added the ``dev_mode`` attribute for the new :ref:`Python Development + Mode ` and the ``utf8_mode`` attribute for the new :option:`-X` + ``utf8`` flag. .. data:: float_info - A :term:`struct sequence` holding information about the float type. It + A :term:`named tuple` holding information about the float type. It contains low level information about the precision and internal representation. The values correspond to the various floating-point constants defined in the standard header file :file:`float.h` for the 'C' @@ -398,8 +484,10 @@ always available. +---------------------+----------------+--------------------------------------------------+ | attribute | float.h macro | explanation | +=====================+================+==================================================+ - | :const:`epsilon` | DBL_EPSILON | difference between 1 and the least value greater | - | | | than 1 that is representable as a float | + | :const:`epsilon` | DBL_EPSILON | difference between 1.0 and the least value | + | | | greater than 1.0 that is representable as a float| + | | | | + | | | See also :func:`math.ulp`. | +---------------------+----------------+--------------------------------------------------+ | :const:`dig` | DBL_DIG | maximum number of decimal digits that can be | | | | faithfully represented in a float; see below | @@ -407,20 +495,24 @@ always available. | :const:`mant_dig` | DBL_MANT_DIG | float precision: the number of base-``radix`` | | | | digits in the significand of a float | +---------------------+----------------+--------------------------------------------------+ - | :const:`max` | DBL_MAX | maximum representable finite float | + | :const:`max` | DBL_MAX | maximum representable positive finite float | +---------------------+----------------+--------------------------------------------------+ - | :const:`max_exp` | DBL_MAX_EXP | maximum integer e such that ``radix**(e-1)`` is | + | :const:`max_exp` | DBL_MAX_EXP | maximum integer *e* such that ``radix**(e-1)`` is| | | | a representable finite float | +---------------------+----------------+--------------------------------------------------+ - | :const:`max_10_exp` | DBL_MAX_10_EXP | maximum integer e such that ``10**e`` is in the | + | :const:`max_10_exp` | DBL_MAX_10_EXP | maximum integer *e* such that ``10**e`` is in the| | | | range of representable finite floats | +---------------------+----------------+--------------------------------------------------+ - | :const:`min` | DBL_MIN | minimum positive normalized float | + | :const:`min` | DBL_MIN | minimum representable positive *normalized* float| + | | | | + | | | Use :func:`math.ulp(0.0) ` to get the | + | | | smallest positive *denormalized* representable | + | | | float. | +---------------------+----------------+--------------------------------------------------+ - | :const:`min_exp` | DBL_MIN_EXP | minimum integer e such that ``radix**(e-1)`` is | + | :const:`min_exp` | DBL_MIN_EXP | minimum integer *e* such that ``radix**(e-1)`` is| | | | a normalized float | +---------------------+----------------+--------------------------------------------------+ - | :const:`min_10_exp` | DBL_MIN_10_EXP | minimum integer e such that ``10**e`` is a | + | :const:`min_10_exp` | DBL_MIN_10_EXP | minimum integer *e* such that ``10**e`` is a | | | | normalized float | +---------------------+----------------+--------------------------------------------------+ | :const:`radix` | FLT_RADIX | radix of exponent representation | @@ -490,14 +582,6 @@ always available. .. versionadded:: 3.7 -.. function:: getcheckinterval() - - Return the interpreter's "check interval"; see :func:`setcheckinterval`. - - .. deprecated:: 3.2 - Use :func:`getswitchinterval` instead. - - .. function:: getdefaultencoding() Return the name of the current default string encoding used by the Unicode @@ -595,7 +679,7 @@ always available. additional garbage collector overhead if the object is managed by the garbage collector. - See `recursive sizeof recipe `_ + See `recursive sizeof recipe `_ for an example of using :func:`getsizeof` recursively to find the size of containers and all their contents. @@ -614,6 +698,8 @@ always available. that is deeper than the call stack, :exc:`ValueError` is raised. The default for *depth* is zero, returning the frame at the top of the call stack. + .. audit-event:: sys._getframe "" sys._getframe + .. impl-detail:: This function should be used for internal and specialized purposes only. @@ -722,25 +808,9 @@ always available. for details.) Use it only for debugging purposes. -.. function:: get_coroutine_wrapper() - - Returns ``None``, or a wrapper set by :func:`set_coroutine_wrapper`. - - .. versionadded:: 3.5 - See :pep:`492` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - - .. deprecated:: 3.7 - The coroutine wrapper functionality has been deprecated, and - will be removed in 3.8. See :issue:`32591` for details. - - .. data:: hash_info - A :term:`struct sequence` giving parameters of the numeric hash + A :term:`named tuple` giving parameters of the numeric hash implementation. For more details about hashing of numeric types, see :ref:`numeric-hash`. @@ -788,7 +858,7 @@ always available. This is called ``hexversion`` since it only really looks meaningful when viewed as the result of passing it to the built-in :func:`hex` function. The - :term:`struct sequence` :data:`sys.version_info` may be used for a more + :term:`named tuple` :data:`sys.version_info` may be used for a more human-friendly encoding of the same information. More details of ``hexversion`` can be found at :ref:`apiabiversion`. @@ -833,10 +903,14 @@ always available. .. versionadded:: 3.3 + .. note:: + + The addition of new required attributes must go through the normal PEP + process. See :pep:`421` for more information. .. data:: int_info - A :term:`struct sequence` that holds information about Python's internal + A :term:`named tuple` that holds information about Python's internal representation of integers. The attributes are read only. .. tabularcolumns:: |l|L| @@ -863,6 +937,12 @@ always available. read, so that you can set this hook there. The :mod:`site` module :ref:`sets this `. + .. audit-event:: cpython.run_interactivehook hook sys.__interactivehook__ + + Raises an :ref:`auditing event ` + ``cpython.run_interactivehook`` with the hook object as the argument when + the hook is called on startup. + .. versionadded:: 3.4 @@ -959,6 +1039,16 @@ always available. deleting essential items from the dictionary may cause Python to fail. +.. data:: orig_argv + + The list of the original command line arguments passed to the Python + executable. + + See also :data:`sys.argv`. + + .. versionadded:: 3.10 + + .. data:: path .. index:: triple: module; search; path @@ -1060,6 +1150,28 @@ always available. system's identity. +.. data:: platlibdir + + Name of the platform-specific library directory. It is used to build the + path of standard library and the paths of installed extension modules. + + It is equal to ``"lib"`` on most platforms. On Fedora and SuSE, it is equal + to ``"lib64"`` on 64-bit platforms which gives the following ``sys.path`` + paths (where ``X.Y`` is the Python ``major.minor`` version): + + * ``/usr/lib64/pythonX.Y/``: + Standard library (like ``os.py`` of the :mod:`os` module) + * ``/usr/lib64/pythonX.Y/lib-dynload/``: + C extension modules of the standard library (like the :mod:`errno` module, + the exact filename is platform specific) + * ``/usr/lib/pythonX.Y/site-packages/`` (always use ``lib``, not + :data:`sys.platlibdir`): Third-party modules + * ``/usr/lib64/pythonX.Y/site-packages/``: + C extension modules of third-party packages + + .. versionadded:: 3.9 + + .. data:: prefix A string giving the site-specific directory prefix where the platform @@ -1094,21 +1206,6 @@ always available. implement a dynamic prompt. -.. function:: setcheckinterval(interval) - - Set the interpreter's "check interval". This integer value determines how often - the interpreter checks for periodic things such as thread switches and signal - handlers. The default is ``100``, meaning the check is performed every 100 - Python virtual instructions. Setting it to a larger value may increase - performance for programs using threads. Setting it to a value ``<=`` 0 checks - every virtual instruction, maximizing responsiveness as well as overhead. - - .. deprecated:: 3.2 - This function doesn't have an effect anymore, as the internal logic for - thread switching and asynchronous tasks has been rewritten. Use - :func:`setswitchinterval` instead. - - .. function:: setdlopenflags(n) Set the flags used by the interpreter for :c:func:`dlopen` calls, such as when @@ -1143,6 +1240,8 @@ always available. ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends on the event type. + .. audit-event:: sys.setprofile "" sys.setprofile + The events have the following meaning: ``'call'`` @@ -1204,8 +1303,8 @@ always available. Set the system's trace function, which allows you to implement a Python source code debugger in Python. The function is thread-specific; for a - debugger to support multiple threads, it must be registered using - :func:`settrace` for each thread being debugged. + debugger to support multiple threads, it must register a trace function using + :func:`settrace` for each thread being debugged or use :func:`threading.settrace`. Trace functions should have three arguments: *frame*, *event*, and *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, @@ -1214,7 +1313,8 @@ always available. The trace function is invoked (with *event* set to ``'call'``) whenever a new local scope is entered; it should return a reference to a local trace - function to be used that scope, or ``None`` if the scope shouldn't be traced. + function to be used for the new scope, or ``None`` if the scope shouldn't be + traced. The local trace function should return a reference to itself (or to another function for further tracing in that scope), or ``None`` to turn off tracing @@ -1261,8 +1361,21 @@ always available. Note that as an exception is propagated down the chain of callers, an ``'exception'`` event is generated at each level. + For more fine-grained usage, it's possible to set a trace function by + assigning ``frame.f_trace = tracefunc`` explicitly, rather than relying on + it being set indirectly via the return value from an already installed + trace function. This is also required for activating the trace function on + the current frame, which :func:`settrace` doesn't do. Note that in order + for this to work, a global tracing function must have been installed + with :func:`settrace` in order to enable the runtime tracing machinery, + but it doesn't need to be the same tracing function (e.g. it could be a + low overhead tracing function that simply returns ``None`` to disable + itself immediately on each frame). + For more information on code and frame objects, refer to :ref:`types`. + .. audit-event:: sys.settrace "" sys.settrace + .. impl-detail:: The :func:`settrace` function is intended only for implementing debuggers, @@ -1283,6 +1396,13 @@ always available. first time. The *finalizer* will be called when an asynchronous generator is about to be garbage collected. + .. audit-event:: sys.set_asyncgen_hooks_firstiter "" sys.set_asyncgen_hooks + + .. audit-event:: sys.set_asyncgen_hooks_finalizer "" sys.set_asyncgen_hooks + + Two auditing events are raised because the underlying API consists of two + calls, each of which must raise its own event. + .. versionadded:: 3.6 See :pep:`525` for more details, and for a reference example of a *finalizer* method see the implementation of @@ -1314,49 +1434,6 @@ always available. This function has been added on a provisional basis (see :pep:`411` for details.) Use it only for debugging purposes. -.. function:: set_coroutine_wrapper(wrapper) - - Allows intercepting creation of :term:`coroutine` objects (only ones that - are created by an :keyword:`async def` function; generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine` will not be - intercepted). - - The *wrapper* argument must be either: - - * a callable that accepts one argument (a coroutine object); - * ``None``, to reset the wrapper. - - If called twice, the new wrapper replaces the previous one. The function - is thread-specific. - - The *wrapper* callable cannot define new coroutines directly or indirectly:: - - def wrapper(coro): - async def wrap(coro): - return await coro - return wrap(coro) - sys.set_coroutine_wrapper(wrapper) - - async def foo(): - pass - - # The following line will fail with a RuntimeError, because - # ``wrapper`` creates a ``wrap(coro)`` coroutine: - foo() - - See also :func:`get_coroutine_wrapper`. - - .. versionadded:: 3.5 - See :pep:`492` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - - .. deprecated:: 3.7 - The coroutine wrapper functionality has been deprecated, and - will be removed in 3.8. See :issue:`32591` for details. - .. function:: _enablelegacywindowsfsencoding() Changes the default filesystem encoding and errors mode to 'mbcs' and @@ -1394,7 +1471,7 @@ always available. On Windows, UTF-8 is used for the console device. Non-character devices such as disk files and pipes use the system locale encoding (i.e. the ANSI codepage). Non-console character - devices such as NUL (i.e. where isatty() returns True) use the + devices such as NUL (i.e. where ``isatty()`` returns ``True``) use the value of the console input and output codepages at startup, respectively for stdin and stdout/stderr. This defaults to the system locale encoding if the process is not initially attached @@ -1412,9 +1489,15 @@ always available. for the Windows console, this only applies when :envvar:`PYTHONLEGACYWINDOWSSTDIO` is also set. - * When interactive, ``stdout`` and ``stderr`` streams are line-buffered. - Otherwise, they are block-buffered like regular text files. You can - override this value with the :option:`-u` command-line option. + * When interactive, the ``stdout`` stream is line-buffered. Otherwise, + it is block-buffered like regular text files. The ``stderr`` stream + is line-buffered in both cases. You can make both streams unbuffered + by passing the :option:`-u` command-line option or setting the + :envvar:`PYTHONUNBUFFERED` environment variable. + + .. versionchanged:: 3.9 + Non-interactive ``stderr`` is now line-buffered instead of fully + buffered. .. note:: @@ -1451,7 +1534,7 @@ always available. .. data:: thread_info - A :term:`struct sequence` holding information about the thread + A :term:`named tuple` holding information about the thread implementation. .. tabularcolumns:: |l|p{0.7\linewidth}| @@ -1487,6 +1570,48 @@ always available. is suppressed and only the exception type and value are printed. +.. function:: unraisablehook(unraisable, /) + + Handle an unraisable exception. + + Called when an exception has occurred but there is no way for Python to + handle it. For example, when a destructor raises an exception or during + garbage collection (:func:`gc.collect`). + + The *unraisable* argument has the following attributes: + + * *exc_type*: Exception type. + * *exc_value*: Exception value, can be ``None``. + * *exc_traceback*: Exception traceback, can be ``None``. + * *err_msg*: Error message, can be ``None``. + * *object*: Object causing the exception, can be ``None``. + + The default hook formats *err_msg* and *object* as: + ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message + if *err_msg* is ``None``. + + :func:`sys.unraisablehook` can be overridden to control how unraisable + exceptions are handled. + + Storing *exc_value* using a custom hook can create a reference cycle. It + should be cleared explicitly to break the reference cycle when the + exception is no longer needed. + + Storing *object* using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing *object* after the custom + hook completes to avoid resurrecting objects. + + See also :func:`excepthook` which handles uncaught exceptions. + + .. audit-event:: sys.unraisablehook hook,unraisable sys.unraisablehook + + Raise an auditing event ``sys.unraisablehook`` with arguments + ``hook``, ``unraisable`` when an exception that cannot be handled occurs. + The ``unraisable`` object is the same as what will be passed to the hook. + If no hook has been set, ``hook`` may be ``None``. + + .. versionadded:: 3.8 + .. data:: version A string containing the version number of the Python interpreter plus additional diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index b5a1da80c686d4..78a1dfce9ae05c 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -74,12 +74,12 @@ places. Python currently supports seven schemes: -- *posix_prefix*: scheme for Posix platforms like Linux or Mac OS X. This is +- *posix_prefix*: scheme for POSIX platforms like Linux or Mac OS X. This is the default scheme used when Python or a component is installed. -- *posix_home*: scheme for Posix platforms used when a *home* option is used +- *posix_home*: scheme for POSIX platforms used when a *home* option is used upon installation. This scheme is used when a component is installed through Distutils with a specific home prefix. -- *posix_user*: scheme for Posix platforms used when a component is installed +- *posix_user*: scheme for POSIX platforms used when a component is installed through Distutils and the *user* option is used. This scheme defines paths located under the user home directory. - *nt*: scheme for NT platforms like Windows. diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 7151527ce57a54..d264a3340c98b0 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -31,6 +31,8 @@ The module defines the following functions: If :func:`openlog` has not been called prior to the call to :func:`syslog`, ``openlog()`` will be called with no arguments. + .. audit-event:: syslog.syslog priority,message syslog.syslog + .. function:: openlog([ident[, logoption[, facility]]]) @@ -45,6 +47,8 @@ The module defines the following functions: keyword argument (default is :const:`LOG_USER`) sets the default facility for messages which do not have a facility explicitly encoded. + .. audit-event:: syslog.openlog ident,logoption,facility syslog.openlog + .. versionchanged:: 3.2 In previous versions, keyword arguments were not allowed, and *ident* was required. The default for *ident* was dependent on the system libraries, @@ -60,6 +64,8 @@ The module defines the following functions: :func:`openlog` hasn't already been called), and *ident* and other :func:`openlog` parameters are reset to defaults. + .. audit-event:: syslog.closelog "" syslog.closelog + .. function:: setlogmask(maskpri) @@ -70,6 +76,8 @@ The module defines the following functions: ``LOG_UPTO(pri)`` calculates the mask for all priorities up to and including *pri*. + .. audit-event:: syslog.setlogmask maskpri syslog.setlogmask + The module defines the following constants: Priority levels (high to low): diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index f25af8ca6a338f..cca466b5697948 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -151,6 +151,7 @@ Some facts and figures: .. class:: TarFile + :noindex: Class for reading and writing tar archives. Do not use this class directly: use :func:`tarfile.open` instead. See :ref:`tarfile-objects`. @@ -159,7 +160,10 @@ Some facts and figures: .. function:: is_tarfile(name) Return :const:`True` if *name* is a tar archive file, that the :mod:`tarfile` - module can read. + module can read. *name* may be a :class:`str`, file, or file-like object. + + .. versionchanged:: 3.9 + Support for file and file-like objects. The :mod:`tarfile` module defines the following exceptions: @@ -290,9 +294,10 @@ be finalized; only the internally used file object will be closed. See the *fileobj* is not closed, when :class:`TarFile` is closed. - *format* controls the archive format. It must be one of the constants + *format* controls the archive format for writing. It must be one of the constants :const:`USTAR_FORMAT`, :const:`GNU_FORMAT` or :const:`PAX_FORMAT` that are - defined at module level. + defined at module level. When reading, format will be automatically detected, even + if different formats are present in a single archive. The *tarinfo* argument can be used to replace the default :class:`TarInfo` class with a different one. @@ -783,7 +788,7 @@ How to read a gzip compressed tar archive and display some member information:: import tarfile tar = tarfile.open("sample.tar.gz", "r:gz") for tarinfo in tar: - print(tarinfo.name, "is", tarinfo.size, "bytes in size and is", end="") + print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") if tarinfo.isreg(): print("a regular file.") elif tarinfo.isdir(): diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst index 4ba426425277f9..48a9aea50dddd1 100644 --- a/Doc/library/telnetlib.rst +++ b/Doc/library/telnetlib.rst @@ -29,7 +29,7 @@ Character), EL (Erase Line), GA (Go Ahead), SB (Subnegotiation Begin). .. class:: Telnet(host=None, port=0[, timeout]) :class:`Telnet` represents a connection to a Telnet server. The instance is - initially not connected by default; the :meth:`open` method must be used to + initially not connected by default; the :meth:`~Telnet.open` method must be used to establish a connection. Alternatively, the host name and optional port number can be passed to the constructor too, in which case the connection to the server will be established before the constructor returns. The optional @@ -141,6 +141,8 @@ Telnet Objects Do not try to reopen an already connected instance. + .. audit-event:: telnetlib.Telnet.open self,host,port telnetlib.Telnet.open + .. method:: Telnet.msg(msg, *args) @@ -176,6 +178,8 @@ Telnet Objects block if the connection is blocked. May raise :exc:`OSError` if the connection is closed. + .. audit-event:: telnetlib.Telnet.write self,buffer telnetlib.Telnet.write + .. versionchanged:: 3.3 This method used to raise :exc:`socket.error`, which is now an alias of :exc:`OSError`. diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index daa6f621f1378e..f9421da5fe7dfa 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -31,7 +31,7 @@ is recommended to use keyword arguments for clarity. The module defines the following user-callable items: -.. function:: TemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) +.. function:: TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) Return a :term:`file-like object` that can be used as a temporary storage area. The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon @@ -62,6 +62,8 @@ The module defines the following user-callable items: The :py:data:`os.O_TMPFILE` flag is used if it is available and works (Linux-specific, requires Linux kernel 3.11 or later). + .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile + .. versionchanged:: 3.5 The :py:data:`os.O_TMPFILE` flag is now used if available. @@ -70,7 +72,7 @@ The module defines the following user-callable items: Added *errors* parameter. -.. function:: NamedTemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) +.. function:: NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) This function operates exactly as :func:`TemporaryFile` does, except that the file is guaranteed to have a visible name in the file system (on @@ -85,11 +87,13 @@ The module defines the following user-callable items: attribute is the underlying true file object. This file-like object can be used in a :keyword:`with` statement, just like a normal file. + .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile + .. versionchanged:: 3.8 Added *errors* parameter. -.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) +.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) This function operates exactly as :func:`TemporaryFile` does, except that data is spooled in memory until the file size exceeds *max_size*, or @@ -101,8 +105,8 @@ The module defines the following user-callable items: causes the file to roll over to an on-disk file regardless of its size. The returned object is a file-like object whose :attr:`_file` attribute - is either an :class:`io.BytesIO` or :class:`io.StringIO` object (depending on - whether binary or text *mode* was specified) or a true file + is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object + (depending on whether binary or text *mode* was specified) or a true file object, depending on whether :func:`rollover` has been called. This file-like object can be used in a :keyword:`with` statement, just like a normal file. @@ -130,6 +134,8 @@ The module defines the following user-callable items: The directory can be explicitly cleaned up by calling the :func:`cleanup` method. + .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory + .. versionadded:: 3.2 @@ -169,20 +175,24 @@ The module defines the following user-callable items: If you want to force a bytes return value with otherwise default behavior, pass ``suffix=b''``. - If *text* is specified, it indicates whether to open the file in binary - mode (the default) or text mode. On some platforms, this makes no - difference. + If *text* is specified and true, the file is opened in text mode. + Otherwise, (the default) the file is opened in binary mode. :func:`mkstemp` returns a tuple containing an OS-level handle to an open file (as would be returned by :func:`os.open`) and the absolute pathname of that file, in that order. + .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp + .. versionchanged:: 3.5 *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to obtain a bytes return value. Prior to this, only str was allowed. *suffix* and *prefix* now accept and default to ``None`` to cause an appropriate default value to be used. + .. versionchanged:: 3.6 + The *dir* parameter now accepts a :term:`path-like object`. + .. function:: mkdtemp(suffix=None, prefix=None, dir=None) @@ -198,12 +208,17 @@ The module defines the following user-callable items: :func:`mkdtemp` returns the absolute pathname of the new directory. + .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp + .. versionchanged:: 3.5 *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to obtain a bytes return value. Prior to this, only str was allowed. *suffix* and *prefix* now accept and default to ``None`` to cause an appropriate default value to be used. + .. versionchanged:: 3.6 + The *dir* parameter now accepts a :term:`path-like object`. + .. function:: gettempdir() diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 054521dcc5d3eb..6495b4844449e5 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -247,54 +247,65 @@ The :mod:`test.support` module defines the following constants: Path for shell if not on Windows; otherwise ``None``. -.. data:: FS_NONASCII - - A non-ASCII character encodable by :func:`os.fsencode`. +.. data:: LOOPBACK_TIMEOUT + Timeout in seconds for tests using a network server listening on the network + local loopback interface like ``127.0.0.1``. -.. data:: TESTFN + The timeout is long enough to prevent test failure: it takes into account + that the client and the server can run in different threads or even + different processes. - Set to a name that is safe to use as the name of a temporary file. Any - temporary file that is created should be closed and unlinked (removed). + The timeout should be long enough for :meth:`~socket.socket.connect`, + :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` methods of + :class:`socket.socket`. + Its default value is 5 seconds. -.. data:: TESTFN_UNICODE + See also :data:`INTERNET_TIMEOUT`. - Set to a non-ASCII name for a temporary file. +.. data:: INTERNET_TIMEOUT -.. data:: TESTFN_ENCODING + Timeout in seconds for network requests going to the Internet. - Set to :func:`sys.getfilesystemencoding`. + The timeout is short enough to prevent a test to wait for too long if the + Internet request is blocked for whatever reason. + Usually, a timeout using :data:`INTERNET_TIMEOUT` should not mark a test as + failed, but skip the test instead: see + :func:`~test.support.socket_helper.transient_internet`. -.. data:: TESTFN_UNENCODABLE + Its default value is 1 minute. - Set to a filename (str type) that should not be able to be encoded by file - system encoding in strict mode. It may be ``None`` if it's not possible to - generate such a filename. + See also :data:`LOOPBACK_TIMEOUT`. -.. data:: TESTFN_UNDECODABLE +.. data:: SHORT_TIMEOUT - Set to a filename (bytes type) that should not be able to be decoded by - file system encoding in strict mode. It may be ``None`` if it's not - possible to generate such a filename. + Timeout in seconds to mark a test as failed if the test takes "too long". + The timeout value depends on the regrtest ``--timeout`` command line option. -.. data:: TESTFN_NONASCII + If a test using :data:`SHORT_TIMEOUT` starts to fail randomly on slow + buildbots, use :data:`LONG_TIMEOUT` instead. - Set to a filename containing the :data:`FS_NONASCII` character. + Its default value is 30 seconds. -.. data:: IPV6_ENABLED +.. data:: LONG_TIMEOUT - Set to ``True`` if IPV6 is enabled on this host, ``False`` otherwise. + Timeout in seconds to detect when a test hangs. + It is long enough to reduce the risk of test failure on the slowest Python + buildbots. It should not be used to mark a test as failed if the test takes + "too long". The timeout value depends on the regrtest ``--timeout`` command + line option. -.. data:: SAVEDCWD + Its default value is 5 minutes. - Set to :func:`os.getcwd`. + See also :data:`LOOPBACK_TIMEOUT`, :data:`INTERNET_TIMEOUT` and + :data:`SHORT_TIMEOUT`. .. data:: PGO @@ -356,51 +367,36 @@ The :mod:`test.support` module defines the following constants: Check for presence of docstrings. + .. data:: TEST_HTTP_URL Define the URL of a dedicated HTTP server for the network tests. +.. data:: ALWAYS_EQ -The :mod:`test.support` module defines the following functions: - -.. function:: forget(module_name) - - Remove the module named *module_name* from ``sys.modules`` and delete any - byte-compiled files of the module. - - -.. function:: unload(name) - - Delete *name* from ``sys.modules``. - - -.. function:: unlink(filename) + Object that is equal to anything. Used to test mixed type comparison. - Call :func:`os.unlink` on *filename*. On Windows platforms, this is - wrapped with a wait loop that checks for the existence fo the file. +.. data:: NEVER_EQ -.. function:: rmdir(filename) + Object that is not equal to anything (even to :data:`ALWAYS_EQ`). + Used to test mixed type comparison. - Call :func:`os.rmdir` on *filename*. On Windows platforms, this is - wrapped with a wait loop that checks for the existence of the file. +.. data:: LARGEST -.. function:: rmtree(path) + Object that is greater than anything (except itself). + Used to test mixed type comparison. - Call :func:`shutil.rmtree` on *path* or call :func:`os.lstat` and - :func:`os.rmdir` to remove a path and its contents. On Windows platforms, - this is wrapped with a wait loop that checks for the existence of the files. +.. data:: SMALLEST -.. function:: make_legacy_pyc(source) + Object that is less than anything (except itself). + Used to test mixed type comparison. - Move a :pep:`3147`/:pep:`488` pyc file to its legacy pyc location and return the file - system path to the legacy pyc file. The *source* value is the file system - path to the source file. It does not need to exist, however the PEP - 3147/488 pyc file must exist. +The :mod:`test.support` module defines the following functions: .. function:: is_resource_enabled(resource) @@ -447,16 +443,6 @@ The :mod:`test.support` module defines the following functions: rather than looking directly in the path directories. -.. function:: create_empty_file(filename) - - Create an empty file with *filename*. If it already exists, truncate it. - - -.. function:: fd_count() - - Count the number of open file descriptors. - - .. function:: match_test(test) Match *test* to patterns set in :func:`set_match_tests`. @@ -511,79 +497,6 @@ The :mod:`test.support` module defines the following functions: check_impl_detail(cpython=False) # Everywhere except CPython. -.. function:: check_warnings(\*filters, quiet=True) - - A convenience wrapper for :func:`warnings.catch_warnings()` that makes it - easier to test that a warning was correctly raised. It is approximately - equivalent to calling ``warnings.catch_warnings(record=True)`` with - :meth:`warnings.simplefilter` set to ``always`` and with the option to - automatically validate the results that are recorded. - - ``check_warnings`` accepts 2-tuples of the form ``("message regexp", - WarningCategory)`` as positional arguments. If one or more *filters* are - provided, or if the optional keyword argument *quiet* is ``False``, - it checks to make sure the warnings are as expected: each specified filter - must match at least one of the warnings raised by the enclosed code or the - test fails, and if any warnings are raised that do not match any of the - specified filters the test fails. To disable the first of these checks, - set *quiet* to ``True``. - - If no arguments are specified, it defaults to:: - - check_warnings(("", Warning), quiet=True) - - In this case all warnings are caught and no errors are raised. - - On entry to the context manager, a :class:`WarningRecorder` instance is - returned. The underlying warnings list from - :func:`~warnings.catch_warnings` is available via the recorder object's - :attr:`warnings` attribute. As a convenience, the attributes of the object - representing the most recent warning can also be accessed directly through - the recorder object (see example below). If no warning has been raised, - then any of the attributes that would otherwise be expected on an object - representing a warning will return ``None``. - - The recorder object also has a :meth:`reset` method, which clears the - warnings list. - - The context manager is designed to be used like this:: - - with check_warnings(("assertion is always true", SyntaxWarning), - ("", UserWarning)): - exec('assert(False, "Hey!")') - warnings.warn(UserWarning("Hide me!")) - - In this case if either warning was not raised, or some other warning was - raised, :func:`check_warnings` would raise an error. - - When a test needs to look more deeply into the warnings, rather than - just checking whether or not they occurred, code like this can be used:: - - with check_warnings(quiet=True) as w: - warnings.warn("foo") - assert str(w.args[0]) == "foo" - warnings.warn("bar") - assert str(w.args[0]) == "bar" - assert str(w.warnings[0].args[0]) == "foo" - assert str(w.warnings[1].args[0]) == "bar" - w.reset() - assert len(w.warnings) == 0 - - - Here all warnings will be caught, and the test code tests the captured - warnings directly. - - .. versionchanged:: 3.2 - New optional arguments *filters* and *quiet*. - - -.. function:: check_no_resource_warning(testcase) - - Context manager to check that no :exc:`ResourceWarning` was raised. You - must remove the object which may emit :exc:`ResourceWarning` before the - end of the context manager. - - .. function:: set_memlimit(limit) Set the values for :data:`max_memuse` and :data:`real_max_memuse` for big @@ -602,13 +515,6 @@ The :mod:`test.support` module defines the following functions: ``sys.stdout`` if it's not set. -.. function:: strip_python_strerr(stderr) - - Strip the *stderr* of a Python process from potential debug output - emitted by the interpreter. This will typically be run on the result of - :meth:`subprocess.Popen.communicate`. - - .. function:: args_from_interpreter_flags() Return a list of command line arguments reproducing the current settings @@ -646,53 +552,6 @@ The :mod:`test.support` module defines the following functions: self.assertEqual(captured, "hello") -.. function:: temp_dir(path=None, quiet=False) - - A context manager that creates a temporary directory at *path* and - yields the directory. - - If *path* is ``None``, the temporary directory is created using - :func:`tempfile.mkdtemp`. If *quiet* is ``False``, the context manager - raises an exception on error. Otherwise, if *path* is specified and - cannot be created, only a warning is issued. - - -.. function:: change_cwd(path, quiet=False) - - A context manager that temporarily changes the current working - directory to *path* and yields the directory. - - If *quiet* is ``False``, the context manager raises an exception - on error. Otherwise, it issues only a warning and keeps the current - working directory the same. - - -.. function:: temp_cwd(name='tempcwd', quiet=False) - - A context manager that temporarily creates a new directory and - changes the current working directory (CWD). - - The context manager creates a temporary directory in the current - directory with name *name* before temporarily changing the current - working directory. If *name* is ``None``, the temporary directory is - created using :func:`tempfile.mkdtemp`. - - If *quiet* is ``False`` and it is not possible to create or change - the CWD, an error is raised. Otherwise, only a warning is raised - and the original CWD is used. - - -.. function:: temp_umask(umask) - - A context manager that temporarily sets the process umask. - - -.. function:: transient_internet(resource_name, *, timeout=30.0, errnos=()) - - A context manager that raises :exc:`ResourceDenied` when various issues - with the internet connection manifest themselves as exceptions. - - .. function:: disable_faulthandler() A context manager that replaces ``sys.stderr`` with ``sys.__stderr__``. @@ -748,16 +607,28 @@ The :mod:`test.support` module defines the following functions: target of the "as" clause, if there is one. -.. function:: wait_threads_exit(timeout=60.0) +.. function:: print_warning(msg) - Context manager to wait until all threads created in the ``with`` statement - exit. + Print a warning into :data:`sys.__stderr__`. Format the message as: + ``f"Warning -- {msg}"``. If *msg* is made of multiple lines, add + ``"Warning -- "`` prefix to each line. + .. versionadded:: 3.9 -.. function:: start_threads(threads, unlock=None) - Context manager to start *threads*. It attempts to join the threads upon - exit. +.. function:: wait_process(pid, *, exitcode, timeout=None) + + Wait until process *pid* completes and check that the process exit code is + *exitcode*. + + Raise an :exc:`AssertionError` if the process exit code is not equal to + *exitcode*. + + If the process runs longer than *timeout* seconds (:data:`SHORT_TIMEOUT` by + default), kill the process and raise an :exc:`AssertionError`. The timeout + feature is not available on Windows. + + .. versionadded:: 3.9 .. function:: calcobjsize(fmt) @@ -778,34 +649,6 @@ The :mod:`test.support` module defines the following functions: header size equals *size*. -.. function:: can_symlink() - - Return ``True`` if the OS supports symbolic links, ``False`` - otherwise. - - -.. function:: can_xattr() - - Return ``True`` if the OS supports xattr, ``False`` - otherwise. - - -.. decorator:: skip_unless_symlink - - A decorator for running tests that require support for symbolic links. - - -.. decorator:: skip_unless_xattr - - A decorator for running tests that require support for xattr. - - -.. decorator:: skip_unless_bind_unix_socket - - A decorator for running tests that require a functional bind() for Unix - sockets. - - .. decorator:: anticipate_failure(condition) A decorator to conditionally mark tests with @@ -904,11 +747,6 @@ The :mod:`test.support` module defines the following functions: the trace function. -.. decorator:: reap_threads(func) - - Decorator to ensure the threads are cleaned up even if the test fails. - - .. decorator:: bigmemtest(size, memuse, dry_run=True) Decorator for bigmem tests. @@ -930,12 +768,6 @@ The :mod:`test.support` module defines the following functions: wrap. -.. function:: make_bad_fd() - - Create an invalid file descriptor by opening and closing a temporary file, - and returning its descriptor. - - .. function:: check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None) Test for syntax errors in *statement* by attempting to compile *statement*. @@ -946,162 +778,50 @@ The :mod:`test.support` module defines the following functions: the offset of the exception. -.. function:: check_syntax_warning(testcase, statement, errtext='', *, lineno=1, offset=None) +.. function:: open_urlresource(url, *args, **kw) - Test for syntax warning in *statement* by attempting to compile *statement*. - Test also that the :exc:`SyntaxWarning` is emitted only once, and that it - will be converted to a :exc:`SyntaxError` when turned into error. - *testcase* is the :mod:`unittest` instance for the test. *errtext* is the - regular expression which should match the string representation of the - emitted :exc:`SyntaxWarning` and raised :exc:`SyntaxError`. If *lineno* - is not ``None``, compares to the line of the warning and exception. - If *offset* is not ``None``, compares to the offset of the exception. + Open *url*. If open fails, raises :exc:`TestFailed`. - .. versionadded:: 3.8 +.. function:: reap_children() -.. function:: open_urlresource(url, *args, **kw) + Use this at the end of ``test_main`` whenever sub-processes are started. + This will help ensure that no extra children (zombies) stick around to + hog resources and create problems when looking for refleaks. - Open *url*. If open fails, raises :exc:`TestFailed`. +.. function:: get_attribute(obj, name) -.. function:: import_module(name, deprecated=False, *, required_on()) + Get an attribute, raising :exc:`unittest.SkipTest` if :exc:`AttributeError` + is raised. - This function imports and returns the named module. Unlike a normal - import, this function raises :exc:`unittest.SkipTest` if the module - cannot be imported. - Module and package deprecation messages are suppressed during this import - if *deprecated* is ``True``. If a module is required on a platform but - optional for others, set *required_on* to an iterable of platform prefixes - which will be compared against :data:`sys.platform`. - - .. versionadded:: 3.1 - - -.. function:: import_fresh_module(name, fresh=(), blocked=(), deprecated=False) - - This function imports and returns a fresh copy of the named Python module - by removing the named module from ``sys.modules`` before doing the import. - Note that unlike :func:`reload`, the original module is not affected by - this operation. - - *fresh* is an iterable of additional module names that are also removed - from the ``sys.modules`` cache before doing the import. - - *blocked* is an iterable of module names that are replaced with ``None`` - in the module cache during the import to ensure that attempts to import - them raise :exc:`ImportError`. - - The named module and any modules named in the *fresh* and *blocked* - parameters are saved before starting the import and then reinserted into - ``sys.modules`` when the fresh import is complete. - - Module and package deprecation messages are suppressed during this import - if *deprecated* is ``True``. - - This function will raise :exc:`ImportError` if the named module cannot be - imported. - - Example use:: - - # Get copies of the warnings module for testing without affecting the - # version being used by the rest of the test suite. One copy uses the - # C implementation, the other is forced to use the pure Python fallback - # implementation - py_warnings = import_fresh_module('warnings', blocked=['_warnings']) - c_warnings = import_fresh_module('warnings', fresh=['_warnings']) - - .. versionadded:: 3.1 - - -.. function:: modules_setup() - - Return a copy of :data:`sys.modules`. - - -.. function:: modules_cleanup(oldmodules) - - Remove modules except for *oldmodules* and ``encodings`` in order to - preserve internal cache. - - -.. function:: threading_setup() - - Return current thread count and copy of dangling threads. - - -.. function:: threading_cleanup(*original_values) - - Cleanup up threads not specified in *original_values*. Designed to emit - a warning if a test leaves running threads in the background. - - -.. function:: join_thread(thread, timeout=30.0) +.. function:: catch_unraisable_exception() - Join a *thread* within *timeout*. Raise an :exc:`AssertionError` if thread - is still alive after *timeout* seconds. - - -.. function:: reap_children() - - Use this at the end of ``test_main`` whenever sub-processes are started. - This will help ensure that no extra children (zombies) stick around to - hog resources and create problems when looking for refleaks. - - -.. function:: get_attribute(obj, name) - - Get an attribute, raising :exc:`unittest.SkipTest` if :exc:`AttributeError` - is raised. - - -.. function:: bind_port(sock, host=HOST) - - Bind the socket to a free port and return the port number. Relies on - ephemeral ports in order to ensure we are using an unbound port. This is - important as many tests may be running simultaneously, especially in a - buildbot environment. This method raises an exception if the - ``sock.family`` is :const:`~socket.AF_INET` and ``sock.type`` is - :const:`~socket.SOCK_STREAM`, and the socket has - :const:`~socket.SO_REUSEADDR` or :const:`~socket.SO_REUSEPORT` set on it. - Tests should never set these socket options for TCP/IP sockets. - The only case for setting these options is testing multicasting via - multiple UDP sockets. - - Additionally, if the :const:`~socket.SO_EXCLUSIVEADDRUSE` socket option is - available (i.e. on Windows), it will be set on the socket. This will - prevent anyone else from binding to our host/port for the duration of the - test. + Context manager catching unraisable exception using + :func:`sys.unraisablehook`. + Storing the exception value (``cm.unraisable.exc_value``) creates a + reference cycle. The reference cycle is broken explicitly when the context + manager exits. -.. function:: bind_unix_socket(sock, addr) + Storing the object (``cm.unraisable.object``) can resurrect it if it is set + to an object which is being finalized. Exiting the context manager clears + the stored object. - Bind a unix socket, raising :exc:`unittest.SkipTest` if - :exc:`PermissionError` is raised. + Usage:: + with support.catch_unraisable_exception() as cm: + # code creating an "unraisable exception" + ... -.. function:: find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM) + # check the unraisable exception: use cm.unraisable + ... - Returns an unused port that should be suitable for binding. This is - achieved by creating a temporary socket with the same family and type as - the ``sock`` parameter (default is :const:`~socket.AF_INET`, - :const:`~socket.SOCK_STREAM`), - and binding it to the specified host address (defaults to ``0.0.0.0``) - with the port set to 0, eliciting an unused ephemeral port from the OS. - The temporary socket is then closed and deleted, and the ephemeral port is - returned. + # cm.unraisable attribute no longer exists at this point + # (to break a reference cycle) - Either this method or :func:`bind_port` should be used for any tests - where a server socket needs to be bound to a particular port for the - duration of the test. - Which one to use depends on whether the calling code is creating a Python - socket, or if an unused port needs to be provided in a constructor - or passed to an external program (i.e. the ``-accept`` argument to - openssl's s_server mode). Always prefer :func:`bind_port` over - :func:`find_unused_port` where possible. Using a hard coded port is - discouraged since it can make multiple instances of the test impossible to - run simultaneously, which is a problem for buildbots. + .. versionadded:: 3.8 .. function:: load_package_tests(pkg_dir, loader, standard_tests, pattern) @@ -1119,11 +839,6 @@ The :mod:`test.support` module defines the following functions: return load_package_tests(os.path.dirname(__file__), *args) -.. function:: fs_is_case_insensitive(directory) - - Return ``True`` if the file system for *directory* is case-insensitive. - - .. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()) Returns the set of attributes, functions or methods of *ref_api* not @@ -1163,7 +878,7 @@ The :mod:`test.support` module defines the following functions: missing. -.. function:: check__all__(test_case, module, name_of_module=None, extra=(), blacklist=()) +.. function:: check__all__(test_case, module, name_of_module=None, extra=(), not_exported=()) Assert that the ``__all__`` variable of *module* contains all public names. @@ -1180,8 +895,8 @@ The :mod:`test.support` module defines the following functions: detected as "public", like objects without a proper ``__module__`` attribute. If provided, it will be added to the automatically detected ones. - The *blacklist* argument can be a set of names that must not be treated as part of - the public API even though their names indicate otherwise. + The *not_exported* argument can be a set of names that must not be treated + as part of the public API even though their names indicate otherwise. Example use:: @@ -1197,46 +912,23 @@ The :mod:`test.support` module defines the following functions: class OtherTestCase(unittest.TestCase): def test__all__(self): extra = {'BAR_CONST', 'FOO_CONST'} - blacklist = {'baz'} # Undocumented name. + not_exported = {'baz'} # Undocumented name. # bar imports part of its API from _bar. support.check__all__(self, bar, ('bar', '_bar'), - extra=extra, blacklist=blacklist) + extra=extra, not_exported=not_exported) .. versionadded:: 3.6 +.. function:: skip_if_broken_multiprocessing_synchronize() -The :mod:`test.support` module defines the following classes: - -.. class:: TransientResource(exc, **kwargs) - - Instances are a context manager that raises :exc:`ResourceDenied` if the - specified exception type is raised. Any keyword arguments are treated as - attribute/value pairs to be compared against any exception raised within the - :keyword:`with` statement. Only if all pairs match properly against - attributes on the exception is :exc:`ResourceDenied` raised. - - -.. class:: EnvironmentVarGuard() - - Class used to temporarily set or unset environment variables. Instances can - be used as a context manager and have a complete dictionary interface for - querying/modifying the underlying ``os.environ``. After exit from the - context manager all changes to environment variables done through this - instance will be rolled back. - - .. versionchanged:: 3.1 - Added dictionary interface. - -.. method:: EnvironmentVarGuard.set(envvar, value) - - Temporarily set the environment variable ``envvar`` to the value of - ``value``. - + Skip tests if the :mod:`multiprocessing.synchronize` module is missing, if + there is no available semaphore implementation, or if creating a lock raises + an :exc:`OSError`. -.. method:: EnvironmentVarGuard.unset(envvar) + .. versionadded:: 3.10 - Temporarily unset the environment variable ``envvar``. +The :mod:`test.support` module defines the following classes: .. class:: SuppressCrashReport() @@ -1253,70 +945,107 @@ The :mod:`test.support` module defines the following classes: On both platforms, the old value is restored by :meth:`__exit__`. -.. class:: CleanImport(*module_names) +.. class:: SaveSignals() - A context manager to force import to return a new module reference. This - is useful for testing module-level behaviors, such as the emission of a - DeprecationWarning on import. Example usage:: + Class to save and restore signal handlers registered by the Python signal + handler. - with CleanImport('foo'): - importlib.import_module('foo') # New reference. +.. class:: Matcher() -.. class:: DirsOnSysPath(*paths) + .. method:: matches(self, d, **kwargs) - A context manager to temporarily add directories to sys.path. + Try to match a single dict with the supplied arguments. - This makes a copy of :data:`sys.path`, appends any directories given - as positional arguments, then reverts :data:`sys.path` to the copied - settings when the context ends. - Note that *all* :data:`sys.path` modifications in the body of the - context manager, including replacement of the object, - will be reverted at the end of the block. + .. method:: match_value(self, k, dv, v) + Try to match a single stored value (*dv*) with a supplied value (*v*). -.. class:: SaveSignals() - Class to save and restore signal handlers registered by the Python signal - handler. +.. class:: BasicTestRunner() + .. method:: run(test) -.. class:: Matcher() + Run *test* and return the result. - .. method:: matches(self, d, **kwargs) - Try to match a single dict with the supplied arguments. +:mod:`test.support.socket_helper` --- Utilities for socket tests +================================================================ +.. module:: test.support.socket_helper + :synopsis: Support for socket tests. - .. method:: match_value(self, k, dv, v) - Try to match a single stored value (*dv*) with a supplied value (*v*). +The :mod:`test.support.socket_helper` module provides support for socket tests. +.. versionadded:: 3.9 -.. class:: WarningsRecorder() - Class used to record warnings for unit tests. See documentation of - :func:`check_warnings` above for more details. +.. data:: IPV6_ENABLED + Set to ``True`` if IPv6 is enabled on this host, ``False`` otherwise. -.. class:: BasicTestRunner() - .. method:: run(test) +.. function:: find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM) - Run *test* and return the result. + Returns an unused port that should be suitable for binding. This is + achieved by creating a temporary socket with the same family and type as + the ``sock`` parameter (default is :const:`~socket.AF_INET`, + :const:`~socket.SOCK_STREAM`), + and binding it to the specified host address (defaults to ``0.0.0.0``) + with the port set to 0, eliciting an unused ephemeral port from the OS. + The temporary socket is then closed and deleted, and the ephemeral port is + returned. + Either this method or :func:`bind_port` should be used for any tests + where a server socket needs to be bound to a particular port for the + duration of the test. + Which one to use depends on whether the calling code is creating a Python + socket, or if an unused port needs to be provided in a constructor + or passed to an external program (i.e. the ``-accept`` argument to + openssl's s_server mode). Always prefer :func:`bind_port` over + :func:`find_unused_port` where possible. Using a hard coded port is + discouraged since it can make multiple instances of the test impossible to + run simultaneously, which is a problem for buildbots. -.. class:: TestHandler(logging.handlers.BufferingHandler) - Class for logging support. +.. function:: bind_port(sock, host=HOST) + Bind the socket to a free port and return the port number. Relies on + ephemeral ports in order to ensure we are using an unbound port. This is + important as many tests may be running simultaneously, especially in a + buildbot environment. This method raises an exception if the + ``sock.family`` is :const:`~socket.AF_INET` and ``sock.type`` is + :const:`~socket.SOCK_STREAM`, and the socket has + :const:`~socket.SO_REUSEADDR` or :const:`~socket.SO_REUSEPORT` set on it. + Tests should never set these socket options for TCP/IP sockets. + The only case for setting these options is testing multicasting via + multiple UDP sockets. -.. class:: FakePath(path) + Additionally, if the :const:`~socket.SO_EXCLUSIVEADDRUSE` socket option is + available (i.e. on Windows), it will be set on the socket. This will + prevent anyone else from binding to our host/port for the duration of the + test. + + +.. function:: bind_unix_socket(sock, addr) + + Bind a unix socket, raising :exc:`unittest.SkipTest` if + :exc:`PermissionError` is raised. + + +.. decorator:: skip_unless_bind_unix_socket + + A decorator for running tests that require a functional ``bind()`` for Unix + sockets. - Simple :term:`path-like object`. It implements the :meth:`__fspath__` - method which just returns the *path* argument. If *path* is an exception, - it will be raised in :meth:`!__fspath__`. + +.. function:: transient_internet(resource_name, *, timeout=30.0, errnos=()) + + A context manager that raises :exc:`~test.support.ResourceDenied` when + various issues with the internet connection manifest themselves as + exceptions. :mod:`test.support.script_helper` --- Utilities for the Python execution tests @@ -1354,6 +1083,9 @@ script execution tests. in a subprocess. The values can include ``__isolated``, ``__cleanenv``, ``__cwd``, and ``TERM``. + .. versionchanged:: 3.9 + The function no longer strips whitespaces from *stderr*. + .. function:: assert_python_ok(*args, **env_vars) @@ -1367,6 +1099,9 @@ script execution tests. Python is started in isolated mode (command line option ``-I``), except if the ``__isolated`` keyword is set to ``False``. + .. versionchanged:: 3.9 + The function no longer strips whitespaces from *stderr*. + .. function:: assert_python_failure(*args, **env_vars) @@ -1376,6 +1111,9 @@ script execution tests. See :func:`assert_python_ok` for more options. + .. versionchanged:: 3.9 + The function no longer strips whitespaces from *stderr*. + .. function:: spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw) @@ -1419,3 +1157,513 @@ script execution tests. containing the *source*. If *compiled* is ``True``, both source files will be compiled and added to the zip package. Return a tuple of the full zip path and the archive name for the zip file. + + +:mod:`test.support.bytecode_helper` --- Support tools for testing correct bytecode generation +============================================================================================= + +.. module:: test.support.bytecode_helper + :synopsis: Support tools for testing correct bytecode generation. + +The :mod:`test.support.bytecode_helper` module provides support for testing +and inspecting bytecode generation. + +The module defines the following class: + +.. class:: BytecodeTestCase(unittest.TestCase) + + This class has custom assertion methods for inspecting bytecode. + +.. method:: BytecodeTestCase.get_disassembly_as_string(co) + + Return the disassembly of *co* as string. + + +.. method:: BytecodeTestCase.assertInBytecode(x, opname, argval=_UNSPECIFIED) + + Return instr if *opname* is found, otherwise throws :exc:`AssertionError`. + + +.. method:: BytecodeTestCase.assertNotInBytecode(x, opname, argval=_UNSPECIFIED) + + Throws :exc:`AssertionError` if *opname* is found. + + +:mod:`test.support.threading_helper` --- Utilities for threading tests +====================================================================== + +.. module:: test.support.threading_helper + :synopsis: Support for threading tests. + +The :mod:`test.support.threading_helper` module provides support for threading tests. + +.. versionadded:: 3.10 + + +.. function:: join_thread(thread, timeout=None) + + Join a *thread* within *timeout*. Raise an :exc:`AssertionError` if thread + is still alive after *timeout* seconds. + + +.. decorator:: reap_threads(func) + + Decorator to ensure the threads are cleaned up even if the test fails. + + +.. function:: start_threads(threads, unlock=None) + + Context manager to start *threads*. It attempts to join the threads upon + exit. + + +.. function:: threading_cleanup(*original_values) + + Cleanup up threads not specified in *original_values*. Designed to emit + a warning if a test leaves running threads in the background. + + +.. function:: threading_setup() + + Return current thread count and copy of dangling threads. + + +.. function:: wait_threads_exit(timeout=None) + + Context manager to wait until all threads created in the ``with`` statement + exit. + + +.. function:: catch_threading_exception() + + Context manager catching :class:`threading.Thread` exception using + :func:`threading.excepthook`. + + Attributes set when an exception is catched: + + * ``exc_type`` + * ``exc_value`` + * ``exc_traceback`` + * ``thread`` + + See :func:`threading.excepthook` documentation. + + These attributes are deleted at the context manager exit. + + Usage:: + + with threading_helper.catch_threading_exception() as cm: + # code spawning a thread which raises an exception + ... + + # check the thread exception, use cm attributes: + # exc_type, exc_value, exc_traceback, thread + ... + + # exc_type, exc_value, exc_traceback, thread attributes of cm no longer + # exists at this point + # (to avoid reference cycles) + + .. versionadded:: 3.8 + + +:mod:`test.support.os_helper` --- Utilities for os tests +======================================================================== + +.. module:: test.support.os_helper + :synopsis: Support for os tests. + +The :mod:`test.support.os_helper` module provides support for os tests. + +.. versionadded:: 3.10 + + +.. data:: FS_NONASCII + + A non-ASCII character encodable by :func:`os.fsencode`. + + +.. data:: SAVEDCWD + + Set to :func:`os.getcwd`. + + +.. data:: TESTFN + + Set to a name that is safe to use as the name of a temporary file. Any + temporary file that is created should be closed and unlinked (removed). + + +.. data:: TESTFN_NONASCII + + Set to a filename containing the :data:`FS_NONASCII` character. + + +.. data:: TESTFN_UNENCODABLE + + Set to a filename (str type) that should not be able to be encoded by file + system encoding in strict mode. It may be ``None`` if it's not possible to + generate such a filename. + + +.. data:: TESTFN_UNDECODABLE + + Set to a filename (bytes type) that should not be able to be decoded by + file system encoding in strict mode. It may be ``None`` if it's not + possible to generate such a filename. + + +.. data:: TESTFN_UNICODE + + Set to a non-ASCII name for a temporary file. + + +.. class:: EnvironmentVarGuard() + + Class used to temporarily set or unset environment variables. Instances can + be used as a context manager and have a complete dictionary interface for + querying/modifying the underlying ``os.environ``. After exit from the + context manager all changes to environment variables done through this + instance will be rolled back. + + .. versionchanged:: 3.1 + Added dictionary interface. + + +.. class:: FakePath(path) + + Simple :term:`path-like object`. It implements the :meth:`__fspath__` + method which just returns the *path* argument. If *path* is an exception, + it will be raised in :meth:`!__fspath__`. + + +.. method:: EnvironmentVarGuard.set(envvar, value) + + Temporarily set the environment variable ``envvar`` to the value of + ``value``. + + +.. method:: EnvironmentVarGuard.unset(envvar) + + Temporarily unset the environment variable ``envvar``. + + +.. function:: can_symlink() + + Return ``True`` if the OS supports symbolic links, ``False`` + otherwise. + + +.. function:: can_xattr() + + Return ``True`` if the OS supports xattr, ``False`` + otherwise. + + +.. function:: change_cwd(path, quiet=False) + + A context manager that temporarily changes the current working + directory to *path* and yields the directory. + + If *quiet* is ``False``, the context manager raises an exception + on error. Otherwise, it issues only a warning and keeps the current + working directory the same. + + +.. function:: create_empty_file(filename) + + Create an empty file with *filename*. If it already exists, truncate it. + + +.. function:: fd_count() + + Count the number of open file descriptors. + + +.. function:: fs_is_case_insensitive(directory) + + Return ``True`` if the file system for *directory* is case-insensitive. + + +.. function:: make_bad_fd() + + Create an invalid file descriptor by opening and closing a temporary file, + and returning its descriptor. + + +.. function:: rmdir(filename) + + Call :func:`os.rmdir` on *filename*. On Windows platforms, this is + wrapped with a wait loop that checks for the existence of the file. + + +.. function:: rmtree(path) + + Call :func:`shutil.rmtree` on *path* or call :func:`os.lstat` and + :func:`os.rmdir` to remove a path and its contents. On Windows platforms, + this is wrapped with a wait loop that checks for the existence of the files. + + +.. decorator:: skip_unless_symlink + + A decorator for running tests that require support for symbolic links. + + +.. decorator:: skip_unless_xattr + + A decorator for running tests that require support for xattr. + + +.. function:: temp_cwd(name='tempcwd', quiet=False) + + A context manager that temporarily creates a new directory and + changes the current working directory (CWD). + + The context manager creates a temporary directory in the current + directory with name *name* before temporarily changing the current + working directory. If *name* is ``None``, the temporary directory is + created using :func:`tempfile.mkdtemp`. + + If *quiet* is ``False`` and it is not possible to create or change + the CWD, an error is raised. Otherwise, only a warning is raised + and the original CWD is used. + + +.. function:: temp_dir(path=None, quiet=False) + + A context manager that creates a temporary directory at *path* and + yields the directory. + + If *path* is ``None``, the temporary directory is created using + :func:`tempfile.mkdtemp`. If *quiet* is ``False``, the context manager + raises an exception on error. Otherwise, if *path* is specified and + cannot be created, only a warning is issued. + + +.. function:: temp_umask(umask) + + A context manager that temporarily sets the process umask. + + +.. function:: unlink(filename) + + Call :func:`os.unlink` on *filename*. On Windows platforms, this is + wrapped with a wait loop that checks for the existence fo the file. + + +:mod:`test.support.import_helper` --- Utilities for import tests +================================================================ + +.. module:: test.support.import_helper + :synopsis: Support for import tests. + +The :mod:`test.support.import_helper` module provides support for import tests. + +.. versionadded:: 3.10 + + +.. function:: forget(module_name) + + Remove the module named *module_name* from ``sys.modules`` and delete any + byte-compiled files of the module. + + +.. function:: import_fresh_module(name, fresh=(), blocked=(), deprecated=False) + + This function imports and returns a fresh copy of the named Python module + by removing the named module from ``sys.modules`` before doing the import. + Note that unlike :func:`reload`, the original module is not affected by + this operation. + + *fresh* is an iterable of additional module names that are also removed + from the ``sys.modules`` cache before doing the import. + + *blocked* is an iterable of module names that are replaced with ``None`` + in the module cache during the import to ensure that attempts to import + them raise :exc:`ImportError`. + + The named module and any modules named in the *fresh* and *blocked* + parameters are saved before starting the import and then reinserted into + ``sys.modules`` when the fresh import is complete. + + Module and package deprecation messages are suppressed during this import + if *deprecated* is ``True``. + + This function will raise :exc:`ImportError` if the named module cannot be + imported. + + Example use:: + + # Get copies of the warnings module for testing without affecting the + # version being used by the rest of the test suite. One copy uses the + # C implementation, the other is forced to use the pure Python fallback + # implementation + py_warnings = import_fresh_module('warnings', blocked=['_warnings']) + c_warnings = import_fresh_module('warnings', fresh=['_warnings']) + + .. versionadded:: 3.1 + + +.. function:: import_module(name, deprecated=False, *, required_on()) + + This function imports and returns the named module. Unlike a normal + import, this function raises :exc:`unittest.SkipTest` if the module + cannot be imported. + + Module and package deprecation messages are suppressed during this import + if *deprecated* is ``True``. If a module is required on a platform but + optional for others, set *required_on* to an iterable of platform prefixes + which will be compared against :data:`sys.platform`. + + .. versionadded:: 3.1 + + +.. function:: modules_setup() + + Return a copy of :data:`sys.modules`. + + +.. function:: modules_cleanup(oldmodules) + + Remove modules except for *oldmodules* and ``encodings`` in order to + preserve internal cache. + + +.. function:: unload(name) + + Delete *name* from ``sys.modules``. + + +.. function:: make_legacy_pyc(source) + + Move a :pep:`3147`/:pep:`488` pyc file to its legacy pyc location and return the file + system path to the legacy pyc file. The *source* value is the file system + path to the source file. It does not need to exist, however the PEP + 3147/488 pyc file must exist. + + +.. class:: CleanImport(*module_names) + + A context manager to force import to return a new module reference. This + is useful for testing module-level behaviors, such as the emission of a + DeprecationWarning on import. Example usage:: + + with CleanImport('foo'): + importlib.import_module('foo') # New reference. + + +.. class:: DirsOnSysPath(*paths) + + A context manager to temporarily add directories to sys.path. + + This makes a copy of :data:`sys.path`, appends any directories given + as positional arguments, then reverts :data:`sys.path` to the copied + settings when the context ends. + + Note that *all* :data:`sys.path` modifications in the body of the + context manager, including replacement of the object, + will be reverted at the end of the block. + + +:mod:`test.support.warnings_helper` --- Utilities for warnings tests +==================================================================== + +.. module:: test.support.warnings_helper + :synopsis: Support for warnings tests. + +The :mod:`test.support.warnings_helper` module provides support for warnings tests. + +.. versionadded:: 3.10 + + +.. function:: check_no_resource_warning(testcase) + + Context manager to check that no :exc:`ResourceWarning` was raised. You + must remove the object which may emit :exc:`ResourceWarning` before the + end of the context manager. + + +.. function:: check_syntax_warning(testcase, statement, errtext='', *, lineno=1, offset=None) + + Test for syntax warning in *statement* by attempting to compile *statement*. + Test also that the :exc:`SyntaxWarning` is emitted only once, and that it + will be converted to a :exc:`SyntaxError` when turned into error. + *testcase* is the :mod:`unittest` instance for the test. *errtext* is the + regular expression which should match the string representation of the + emitted :exc:`SyntaxWarning` and raised :exc:`SyntaxError`. If *lineno* + is not ``None``, compares to the line of the warning and exception. + If *offset* is not ``None``, compares to the offset of the exception. + + .. versionadded:: 3.8 + + +.. function:: check_warnings(\*filters, quiet=True) + + A convenience wrapper for :func:`warnings.catch_warnings()` that makes it + easier to test that a warning was correctly raised. It is approximately + equivalent to calling ``warnings.catch_warnings(record=True)`` with + :meth:`warnings.simplefilter` set to ``always`` and with the option to + automatically validate the results that are recorded. + + ``check_warnings`` accepts 2-tuples of the form ``("message regexp", + WarningCategory)`` as positional arguments. If one or more *filters* are + provided, or if the optional keyword argument *quiet* is ``False``, + it checks to make sure the warnings are as expected: each specified filter + must match at least one of the warnings raised by the enclosed code or the + test fails, and if any warnings are raised that do not match any of the + specified filters the test fails. To disable the first of these checks, + set *quiet* to ``True``. + + If no arguments are specified, it defaults to:: + + check_warnings(("", Warning), quiet=True) + + In this case all warnings are caught and no errors are raised. + + On entry to the context manager, a :class:`WarningRecorder` instance is + returned. The underlying warnings list from + :func:`~warnings.catch_warnings` is available via the recorder object's + :attr:`warnings` attribute. As a convenience, the attributes of the object + representing the most recent warning can also be accessed directly through + the recorder object (see example below). If no warning has been raised, + then any of the attributes that would otherwise be expected on an object + representing a warning will return ``None``. + + The recorder object also has a :meth:`reset` method, which clears the + warnings list. + + The context manager is designed to be used like this:: + + with check_warnings(("assertion is always true", SyntaxWarning), + ("", UserWarning)): + exec('assert(False, "Hey!")') + warnings.warn(UserWarning("Hide me!")) + + In this case if either warning was not raised, or some other warning was + raised, :func:`check_warnings` would raise an error. + + When a test needs to look more deeply into the warnings, rather than + just checking whether or not they occurred, code like this can be used:: + + with check_warnings(quiet=True) as w: + warnings.warn("foo") + assert str(w.args[0]) == "foo" + warnings.warn("bar") + assert str(w.args[0]) == "bar" + assert str(w.warnings[0].args[0]) == "foo" + assert str(w.warnings[1].args[0]) == "bar" + w.reset() + assert len(w.warnings) == 0 + + + Here all warnings will be caught, and the test code tests the captured + warnings directly. + + .. versionchanged:: 3.2 + New optional arguments *filters* and *quiet*. + + +.. class:: WarningsRecorder() + + Class used to record warnings for unit tests. See documentation of + :func:`check_warnings` above for more details. diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index d254466c9a320d..16837104b6cebc 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -77,6 +77,9 @@ functions should be good enough; otherwise, you should use an instance of equal: the lines ``" hello"`` and ``"\thello"`` are considered to have no common leading whitespace. + Lines containing only whitespace are ignored in the input and normalized to a + single newline character in the output. + For example:: def test(): @@ -236,7 +239,7 @@ hyphenated words; only then will long words be broken if necessary, unless :attr:`fix_sentence_endings` is false by default. Since the sentence detection algorithm relies on ``string.lowercase`` for - the definition of "lowercase letter," and a convention of using two spaces + the definition of "lowercase letter", and a convention of using two spaces after a period to separate sentences on the same line, it is specific to English-language texts. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index d7dbcb107ddadb..7fcf93d74610eb 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -38,6 +38,40 @@ This module defines the following functions: returned. +.. function:: excepthook(args, /) + + Handle uncaught exception raised by :func:`Thread.run`. + + The *args* argument has the following attributes: + + * *exc_type*: Exception type. + * *exc_value*: Exception value, can be ``None``. + * *exc_traceback*: Exception traceback, can be ``None``. + * *thread*: Thread which raised the exception, can be ``None``. + + If *exc_type* is :exc:`SystemExit`, the exception is silently ignored. + Otherwise, the exception is printed out on :data:`sys.stderr`. + + If this function raises an exception, :func:`sys.excepthook` is called to + handle it. + + :func:`threading.excepthook` can be overridden to control how uncaught + exceptions raised by :func:`Thread.run` are handled. + + Storing *exc_value* using a custom hook can create a reference cycle. It + should be cleared explicitly to break the reference cycle when the + exception is no longer needed. + + Storing *thread* using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing *thread* after the custom + hook completes to avoid resurrecting objects. + + .. seealso:: + :func:`sys.excepthook` handles uncaught exceptions. + + .. versionadded:: 3.8 + + .. function:: get_ident() Return the 'thread identifier' of the current thread. This is a nonzero @@ -49,6 +83,18 @@ This module defines the following functions: .. versionadded:: 3.3 +.. function:: get_native_id() + + Return the native integral Thread ID of the current thread assigned by the kernel. + This is a non-negative integer. + Its value may be used to uniquely identify this particular thread system-wide + (until the thread terminates, after which the value may be recycled by the OS). + + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. + + .. versionadded:: 3.8 + + .. function:: enumerate() Return a list of all :class:`Thread` objects currently alive. The list @@ -179,6 +225,10 @@ called is terminated. A thread has a name. The name can be passed to the constructor, and read or changed through the :attr:`~Thread.name` attribute. +If the :meth:`~Thread.run` method raises an exception, +:func:`threading.excepthook` is called to handle it. By default, +:func:`threading.excepthook` ignores silently :exc:`SystemExit`. + A thread can be flagged as a "daemon thread". The significance of this flag is that the entire Python program exits when only daemon threads are left. The initial value is inherited from the creating thread. The flag can be set @@ -250,7 +300,7 @@ since it is impossible to detect the termination of alien threads. You may override this method in a subclass. The standard :meth:`run` method invokes the callable object passed to the object's constructor as - the *target* argument, if any, with sequential and keyword arguments taken + the *target* argument, if any, with positional and keyword arguments taken from the *args* and *kwargs* arguments, respectively. .. method:: join(timeout=None) @@ -297,6 +347,25 @@ since it is impossible to detect the termination of alien threads. another thread is created. The identifier is available even after the thread has exited. + .. attribute:: native_id + + The Thread ID (``TID``) of this thread, as assigned by the OS (kernel). + This is a non-negative integer, or ``None`` if the thread has not + been started. See the :func:`get_native_id` function. + This value may be used to uniquely identify this particular thread + system-wide (until the thread terminates, after which the value + may be recycled by the OS). + + .. note:: + + Similar to Process IDs, Thread IDs are only valid (guaranteed unique + system-wide) from the time the thread is created until the thread + has been terminated. + + .. availability:: Requires :func:`get_native_id` function. + + .. versionadded:: 3.8 + .. method:: is_alive() Return whether the thread is alive. @@ -325,7 +394,8 @@ since it is impossible to detect the termination of alien threads. .. impl-detail:: - In CPython, due to the :term:`Global Interpreter Lock`, only one thread + In CPython, due to the :term:`Global Interpreter Lock + `, only one thread can execute Python code at once (even though certain performance-oriented libraries might overcome this limitation). If you want your application to make better use of the computational @@ -418,6 +488,11 @@ All methods are executed atomically. There is no return value. + .. method:: locked() + + Return true if the lock is acquired. + + .. _rlock-objects: @@ -465,15 +540,15 @@ Reentrant locks also support the :ref:`context management protocol ` There is no return value in this case. When invoked with the *blocking* argument set to true, do the same thing as when - called without arguments, and return true. + called without arguments, and return ``True``. When invoked with the *blocking* argument set to false, do not block. If a call - without an argument would block, return false immediately; otherwise, do the - same thing as when called without arguments, and return true. + without an argument would block, return ``False`` immediately; otherwise, do the + same thing as when called without arguments, and return ``True``. When invoked with the floating-point *timeout* argument set to a positive value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. Return true if the lock has + and as long as the lock cannot be acquired. Return ``True`` if the lock has been acquired, false if the timeout has elapsed. .. versionchanged:: 3.2 @@ -706,29 +781,32 @@ Semaphores also support the :ref:`context management protocol `. When invoked without arguments: * If the internal counter is larger than zero on entry, decrement it by - one and return true immediately. + one and return ``True`` immediately. * If the internal counter is zero on entry, block until awoken by a call to :meth:`~Semaphore.release`. Once awoken (and the counter is greater - than 0), decrement the counter by 1 and return true. Exactly one + than 0), decrement the counter by 1 and return ``True``. Exactly one thread will be awoken by each call to :meth:`~Semaphore.release`. The order in which threads are awoken should not be relied on. When invoked with *blocking* set to false, do not block. If a call - without an argument would block, return false immediately; otherwise, do - the same thing as when called without arguments, and return true. + without an argument would block, return ``False`` immediately; otherwise, do + the same thing as when called without arguments, and return ``True``. When invoked with a *timeout* other than ``None``, it will block for at most *timeout* seconds. If acquire does not complete successfully in - that interval, return false. Return true otherwise. + that interval, return ``False``. Return ``True`` otherwise. .. versionchanged:: 3.2 The *timeout* parameter is new. - .. method:: release() + .. method:: release(n=1) + + Release a semaphore, incrementing the internal counter by *n*. When it + was zero on entry and other threads are waiting for it to become larger + than zero again, wake up *n* of those threads. - Release a semaphore, incrementing the internal counter by one. When it - was zero on entry and another thread is waiting for it to become larger - than zero again, wake up that thread. + .. versionchanged:: 3.9 + Added the *n* parameter to release multiple waiting threads at once. .. class:: BoundedSemaphore(value=1) @@ -796,7 +874,7 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. .. method:: is_set() - Return true if and only if the internal flag is true. + Return ``True`` if and only if the internal flag is true. .. method:: set() @@ -820,7 +898,7 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. floating point number specifying a timeout for the operation in seconds (or fractions thereof). - This method returns true if and only if the internal flag has been set to + This method returns ``True`` if and only if the internal flag has been set to true, either before the wait call or after the wait starts, so it will always return ``True`` except if a timeout is given and the operation times out. @@ -937,7 +1015,7 @@ As an example, here is a simple way to synchronize a client and server thread:: Return the barrier to the default, empty state. Any threads waiting on it will receive the :class:`BrokenBarrierError` exception. - Note that using this function may can require some external + Note that using this function may require some external synchronization if there are other threads whose state is unknown. If a barrier is broken it may be better to just leave it and create a new one. @@ -945,7 +1023,7 @@ As an example, here is a simple way to synchronize a client and server thread:: Put the barrier into a broken state. This causes any active or future calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. Use - this for example if one of the needs to abort, to avoid deadlocking the + this for example if one of the threads needs to abort, to avoid deadlocking the application. It may be preferable to simply create the barrier with a sensible diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 170f8dc629bfe1..cff6320b526db5 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -42,17 +42,12 @@ An explanation of some terminology and conventions is in order. library; for 32-bit systems, it is typically in 2038. .. index:: - single: Year 2000 - single: Y2K + single: 2-digit years -.. _time-y2kissues: - -* **Year 2000 (Y2K) issues**: Python depends on the platform's C library, which - generally doesn't have year 2000 issues, since all dates and times are - represented internally as seconds since the epoch. Function :func:`strptime` - can parse 2-digit years when given ``%y`` format code. When 2-digit years are - parsed, they are converted according to the POSIX and ISO C standards: values - 69--99 are mapped to 1969--1999, and values 0--68 are mapped to 2000--2068. +* Function :func:`strptime` can parse 2-digit years when given ``%y`` format + code. When 2-digit years are parsed, they are converted according to the POSIX + and ISO C standards: values 69--99 are mapped to 1969--1999, and values 0--68 + are mapped to 2000--2068. .. index:: single: UTC @@ -127,39 +122,18 @@ Functions Convert a tuple or :class:`struct_time` representing a time as returned by :func:`gmtime` or :func:`localtime` to a string of the following - form: ``'Sun Jun 20 23:21:05 1993'``. If *t* is not provided, the current time - as returned by :func:`localtime` is used. Locale information is not used by - :func:`asctime`. + form: ``'Sun Jun 20 23:21:05 1993'``. The day field is two characters long + and is space padded if the day is a single digit, + e.g.: ``'Wed Jun 9 04:26:40 1993'``. + + If *t* is not provided, the current time as returned by :func:`localtime` + is used. Locale information is not used by :func:`asctime`. .. note:: Unlike the C function of the same name, :func:`asctime` does not add a trailing newline. - -.. function:: clock() - - .. index:: - single: CPU time - single: processor time - single: benchmarking - - On Unix, return the current processor time as a floating point number expressed - in seconds. The precision, and in fact the very definition of the meaning of - "processor time", depends on that of the C function of the same name. - - On Windows, this function returns wall-clock seconds elapsed since the first - call to this function, as a floating point number, based on the Win32 function - :c:func:`QueryPerformanceCounter`. The resolution is typically better than one - microsecond. - - .. availability:: Windows, Unix. Not available on VxWorks. - - .. deprecated:: 3.3 - The behaviour of this function depends on the platform: use - :func:`perf_counter` or :func:`process_time` instead, depending on your - requirements, to have a well defined behaviour. - .. function:: pthread_getcpuclockid(thread_id) Return the *clk_id* of the thread-specific CPU-time clock for the specified *thread_id*. @@ -227,10 +201,15 @@ Functions .. function:: ctime([secs]) - Convert a time expressed in seconds since the epoch to a string representing - local time. If *secs* is not provided or :const:`None`, the current time as - returned by :func:`.time` is used. ``ctime(secs)`` is equivalent to - ``asctime(localtime(secs))``. Locale information is not used by :func:`ctime`. + Convert a time expressed in seconds since the epoch to a string of a form: + ``'Sun Jun 20 23:21:05 1993'`` representing local time. The day field + is two characters long and is space padded if the day is a single digit, + e.g.: ``'Wed Jun 9 04:26:40 1993'``. + + If *secs* is not provided or :const:`None`, the current time as + returned by :func:`.time` is used. ``ctime(secs)`` is equivalent to + ``asctime(localtime(secs))``. Locale information is not used by + :func:`ctime`. .. function:: get_clock_info(name) @@ -239,7 +218,6 @@ Functions Supported clock names and the corresponding functions to read their value are: - * ``'clock'``: :func:`time.clock` * ``'monotonic'``: :func:`time.monotonic` * ``'perf_counter'``: :func:`time.perf_counter` * ``'process_time'``: :func:`time.process_time` @@ -632,7 +610,7 @@ Functions .. function:: time_ns() -> int - Similar to :func:`time` but returns time as an integer number of nanoseconds + Similar to :func:`~time.time` but returns time as an integer number of nanoseconds since the epoch_. .. versionadded:: 3.7 @@ -796,6 +774,16 @@ These constants are used as parameters for :func:`clock_getres` and .. versionadded:: 3.7 +.. data:: CLOCK_TAI + + `International Atomic Time `_ + + The system must have a current leap second table in order for this to give + the correct answer. PTP or NTP software can maintain a leap second table. + + .. availability:: Linux. + + .. versionadded:: 3.9 .. data:: CLOCK_THREAD_CPUTIME_ID @@ -827,7 +815,6 @@ These constants are used as parameters for :func:`clock_getres` and .. versionadded:: 3.8 - The following constant is the only parameter that can be sent to :func:`clock_settime`. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 8ca37034f79b7c..668fcb860cea87 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -44,8 +44,12 @@ This can be achieved from the :ref:`python-interface` with:: >>> timeit.timeit('"-".join(map(str, range(100)))', number=10000) 0.23702679807320237 +A callable can also be passed from the :ref:`python-interface`:: -Note however that :mod:`timeit` will automatically determine the number of + >>> timeit.timeit(lambda: "-".join(map(str, range(100))), number=10000) + 0.19665591977536678 + +Note however that :func:`.timeit` will automatically determine the number of repetitions only when the command-line interface is used. In the :ref:`timeit-examples` section you can find more advanced examples. @@ -229,7 +233,7 @@ Where the following options are understood: .. cmdoption:: -u, --unit=U - specify a time unit for timer output; can select nsec, usec, msec, or sec + specify a time unit for timer output; can select nsec, usec, msec, or sec .. versionadded:: 3.5 @@ -247,7 +251,8 @@ quotes and using leading spaces. Multiple :option:`-s` options are treated similarly. If :option:`-n` is not given, a suitable number of loops is calculated by trying -successive powers of 10 until the total time is at least 0.2 seconds. +increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the total +time is at least 0.2 seconds. :func:`default_timer` measurements can be affected by other programs running on the same machine, so the best thing to do when accurate timing is necessary is diff --git a/Doc/library/tk.rst b/Doc/library/tk.rst index 95cd1c7712e7c9..c6c73f057cab16 100644 --- a/Doc/library/tk.rst +++ b/Doc/library/tk.rst @@ -33,14 +33,17 @@ alternatives, see the :ref:`other-gui-packages` section. .. toctree:: tkinter.rst + tkinter.colorchooser.rst + tkinter.font.rst + dialog.rst + tkinter.messagebox.rst + tkinter.scrolledtext.rst + tkinter.dnd.rst tkinter.ttk.rst tkinter.tix.rst - tkinter.scrolledtext.rst idle.rst othergui.rst .. Other sections I have in mind are Tkinter internals - Freezing Tkinter applications - - + Freezing Tkinter applications \ No newline at end of file diff --git a/Doc/library/tk_msg.png b/Doc/library/tk_msg.png new file mode 100644 index 00000000000000..c122d8f8ae5ba9 Binary files /dev/null and b/Doc/library/tk_msg.png differ diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst new file mode 100644 index 00000000000000..60f4d707270d64 --- /dev/null +++ b/Doc/library/tkinter.colorchooser.rst @@ -0,0 +1,29 @@ +:mod:`tkinter.colorchooser` --- Color choosing dialog +===================================================== + +.. module:: tkinter.colorchooser + :platform: Tk + :synopsis: Color choosing dialog + +**Source code:** :source:`Lib/tkinter/colorchooser.py` + +-------------- + +The :mod:`tkinter.colorchooser` module provides the :class:`Chooser` class +as an interface to the native color picker dialog. ``Chooser`` implements +a modal color choosing dialog window. The ``Chooser`` class inherits from +the :class:`~tkinter.commondialog.Dialog` class. + +.. class:: Chooser(master=None, **options) + +.. function:: askcolor(color=None, **options) + + Create a color choosing dialog. A call to this method will show the window, + wait for the user to make a selection, and return the selected color (or + ``None``) to the caller. + + +.. seealso:: + + Module :mod:`tkinter.commondialog` + Tkinter standard dialog module \ No newline at end of file diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst new file mode 100644 index 00000000000000..6c11c739e1fa11 --- /dev/null +++ b/Doc/library/tkinter.dnd.rst @@ -0,0 +1,64 @@ +:mod:`tkinter.dnd` --- Drag and drop support +============================================ + +.. module:: tkinter.dnd + :platform: Tk + :synopsis: Tkinter drag-and-drop interface + +**Source code:** :source:`Lib/tkinter/dnd.py` + +-------------- + +.. note:: This is experimental and due to be deprecated when it is replaced + with the Tk DND. + +The :mod:`tkinter.dnd` module provides drag-and-drop support for objects within +a single application, within the same window or between windows. To enable an +object to be dragged, you must create an event binding for it that starts the +drag-and-drop process. Typically, you bind a ButtonPress event to a callback +function that you write (see :ref:`Bindings-and-Events`). The function should +call :func:`dnd_start`, where 'source' is the object to be dragged, and 'event' +is the event that invoked the call (the argument to your callback function). + +Selection of a target object occurs as follows: + +#. Top-down search of area under mouse for target widget + + * Target widget should have a callable *dnd_accept* attribute + * If *dnd_accept* is not present or returns None, search moves to parent widget + * If no target widget is found, then the target object is None + +2. Call to *.dnd_leave(source, event)* +#. Call to *.dnd_enter(source, event)* +#. Call to *.dnd_commit(source, event)* to notify of drop +#. Call to *.dnd_end(target, event)* to signal end of drag-and-drop + + +.. class:: DndHandler(source, event) + + The *DndHandler* class handles drag-and-drop events tracking Motion and + ButtonRelease events on the root of the event widget. + + .. method:: cancel(event=None) + + Cancel the drag-and-drop process. + + .. method:: finish(event, commit=0) + + Execute end of drag-and-drop functions. + + .. method:: on_motion(event) + + Inspect area below mouse for target objects while drag is performed. + + .. method:: on_release(event) + + Signal end of drag when the release pattern is triggered. + +.. function:: dnd_start(source, event) + + Factory function for drag-and-drop process. + +.. seealso:: + + :ref:`Bindings-and-Events` \ No newline at end of file diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst new file mode 100644 index 00000000000000..b0f4505e9e3c69 --- /dev/null +++ b/Doc/library/tkinter.font.rst @@ -0,0 +1,96 @@ +:mod:`tkinter.font` --- Tkinter font wrapper +============================================ + +.. module:: tkinter.font + :platform: Tk + :synopsis: Tkinter font-wrapping class + +**Source code:** :source:`Lib/tkinter/font.py` + +-------------- + +The :mod:`tkinter.font` module provides the :class:`Font` class for creating +and using named fonts. + +The different font weights and slants are: + +.. data:: NORMAL + BOLD + ITALIC + ROMAN + +.. class:: Font(root=None, font=None, name=None, exists=False, **options) + + The :class:`Font` class represents a named font. *Font* instances are given + unique names and can be specified by their family, size, and style + configuration. Named fonts are Tk's method of creating and identifying + fonts as a single object, rather than specifying a font by its attributes + with each occurrence. + + arguments: + + | *font* - font specifier tuple (family, size, options) + | *name* - unique font name + | *exists* - self points to existing named font if true + + additional keyword options (ignored if *font* is specified): + + | *family* - font family i.e. Courier, Times + | *size* - font size + | If *size* is positive it is interpreted as size in points. + | If *size* is a negative number its absolute value is treated + | as size in pixels. + | *weight* - font emphasis (NORMAL, BOLD) + | *slant* - ROMAN, ITALIC + | *underline* - font underlining (0 - none, 1 - underline) + | *overstrike* - font strikeout (0 - none, 1 - strikeout) + + .. method:: actual(option=None, displayof=None) + + Return the attributes of the font. + + .. method:: cget(option) + + Retrieve an attribute of the font. + + .. method:: config(**options) + + Modify attributes of the font. + + .. method:: copy() + + Return new instance of the current font. + + .. method:: measure(text, displayof=None) + + Return amount of space the text would occupy on the specified display + when formatted in the current font. If no display is specified then the + main application window is assumed. + + .. method:: metrics(*options, **kw) + + Return font-specific data. + Options include: + + *ascent* - distance between baseline and highest point that a + character of the font can occupy + + *descent* - distance between baseline and lowest point that a + character of the font can occupy + + *linespace* - minimum vertical separation necessary between any two + characters of the font that ensures no vertical overlap between lines. + + *fixed* - 1 if font is fixed-width else 0 + +.. function:: families(root=None, displayof=None) + + Return the different font families. + +.. function:: names(root=None) + + Return the names of defined fonts. + +.. function:: nametofont(name) + + Return a :class:`Font` representation of a tk named font. \ No newline at end of file diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst new file mode 100644 index 00000000000000..872e72f7a7e2b8 --- /dev/null +++ b/Doc/library/tkinter.messagebox.rst @@ -0,0 +1,39 @@ +:mod:`tkinter.messagebox` --- Tkinter message prompts +===================================================== + +.. module:: tkinter.messagebox + :platform: Tk + :synopsis: Various types of alert dialogs + +**Source code:** :source:`Lib/tkinter/messagebox.py` + +-------------- + +The :mod:`tkinter.messagebox` module provides a template base class as well as +a variety of convenience methods for commonly used configurations. The message +boxes are modal and will return a subset of (True, False, OK, None, Yes, No) based on +the user's selection. Common message box styles and layouts include but are not +limited to: + +.. figure:: tk_msg.png + +.. class:: Message(master=None, **options) + + Create a default information message box. + +**Information message box** + +.. method:: showinfo(title=None, message=None, **options) + +**Warning message boxes** + +.. method:: showwarning(title=None, message=None, **options) + showerror(title=None, message=None, **options) + +**Question message boxes** + +.. method:: askquestion(title=None, message=None, **options) + askokcancel(title=None, message=None, **options) + askretrycancel(title=None, message=None, **options) + askyesno(title=None, message=None, **options) + askyesnocancel(title=None, message=None, **options) \ No newline at end of file diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 60cf892e0888b7..7739f2f60a7980 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -31,7 +31,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. `TKDocs `_ Extensive tutorial plus friendlier widget pages for some of the widgets. - `Tkinter reference: a GUI for Python `_ + `Tkinter 8.5 reference: a GUI for Python `_ On-line reference material. `Tkinter docs from effbot `_ @@ -41,7 +41,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. Book by Mark Lutz, has excellent coverage of Tkinter. `Modern Tkinter for Busy Python Developers `_ - Book by Mark Rozerman about building attractive and modern graphical user interfaces with Python and Tkinter. + Book by Mark Roseman about building attractive and modern graphical user interfaces with Python and Tkinter. `Python and Tkinter Programming `_ Book by John Grayson (ISBN 1-884777-81-3). @@ -55,7 +55,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. `Tcl/Tk recent man pages `_ Recent Tcl/Tk manuals on www.tcl.tk. - `ActiveState Tcl Home Page `_ + `ActiveState Tcl Home Page `_ The Tk/Tcl development is largely taking place at ActiveState. `Tcl and the Tk Toolkit `_ @@ -109,9 +109,6 @@ Or, more often:: Other modules that provide Tk support include: -:mod:`tkinter.scrolledtext` - Text widget with a vertical scroll bar built in. - :mod:`tkinter.colorchooser` Dialog to let the user choose a color. @@ -127,6 +124,9 @@ Other modules that provide Tk support include: :mod:`tkinter.messagebox` Access to standard Tk dialog boxes. +:mod:`tkinter.scrolledtext` + Text widget with a vertical scroll bar built in. + :mod:`tkinter.simpledialog` Basic dialogs and convenience functions. @@ -464,12 +464,11 @@ The Packer .. index:: single: packing (widgets) The packer is one of Tk's geometry-management mechanisms. Geometry managers -are used to specify the relative positioning of the positioning of widgets -within their container - their mutual *master*. In contrast to the more -cumbersome *placer* (which is used less commonly, and we do not cover here), the -packer takes qualitative relationship specification - *above*, *to the left of*, -*filling*, etc - and works everything out to determine the exact placement -coordinates for you. +are used to specify the relative positioning of widgets within their container - +their mutual *master*. In contrast to the more cumbersome *placer* (which is +used less commonly, and we do not cover here), the packer takes qualitative +relationship specification - *above*, *to the left of*, *filling*, etc - and +works everything out to determine the exact placement coordinates for you. The size of any *master* widget is determined by the size of the "slave widgets" inside. The packer is used to control where slave widgets appear inside the @@ -542,31 +541,35 @@ the variable, with no further intervention on your part. For example:: - class App(Frame): - def __init__(self, master=None): + import tkinter as tk + + class App(tk.Frame): + def __init__(self, master): super().__init__(master) self.pack() - self.entrythingy = Entry() + self.entrythingy = tk.Entry() self.entrythingy.pack() - # here is the application variable - self.contents = StringVar() - # set it to some value + # Create the application variable. + self.contents = tk.StringVar() + # Set it to some value. self.contents.set("this is a variable") - # tell the entry widget to watch this variable + # Tell the entry widget to watch this variable. self.entrythingy["textvariable"] = self.contents - # and here we get a callback when the user hits return. - # we will have the program print out the value of the - # application variable when the user hits return + # Define a callback for when the user hits return. + # It prints the current value of the variable. self.entrythingy.bind('', - self.print_contents) + self.print_contents) def print_contents(self, event): - print("hi. contents of entry is now ---->", + print("Hi. The current entry content is:", self.contents.get()) + root = tk.Tk() + myapp = App(root) + myapp.mainloop() The Window Manager ^^^^^^^^^^^^^^^^^^ @@ -680,9 +683,10 @@ scrollcommand This is almost always the :meth:`!set` method of some scrollbar widget, but can be any widget method that takes a single argument. -wrap: +wrap Must be one of: ``"none"``, ``"char"``, or ``"word"``. +.. _Bindings-and-Events: Bindings and Events ^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index 138720e4785f6b..d20365baa38690 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -14,8 +14,7 @@ The :mod:`tkinter.scrolledtext` module provides a class of the same name which implements a basic text widget which has a vertical scroll bar configured to do the "right thing." Using the :class:`ScrolledText` class is a lot easier than -setting up a text widget and scroll bar directly. The constructor is the same -as that of the :class:`tkinter.Text` class. +setting up a text widget and scroll bar directly. The text widget and scrollbar are packed together in a :class:`Frame`, and the methods of the :class:`Grid` and :class:`Pack` geometry managers are acquired @@ -25,12 +24,14 @@ be used directly to achieve most normal geometry management behavior. Should more specific control be necessary, the following attributes are available: +.. class:: ScrolledText(master=None, **kw) -.. attribute:: ScrolledText.frame - The frame which surrounds the text and scroll bar widgets. + .. attribute:: frame + The frame which surrounds the text and scroll bar widgets. -.. attribute:: ScrolledText.vbar - The scroll bar widget. + .. attribute:: vbar + + The scroll bar widget. diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 5ba31feae1444d..6967d7509657b3 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -871,8 +871,8 @@ widget commands. | | remaining values are assumed empty. If there are more values | | | than columns, the extra values are ignored. | +--------+---------------------------------------------------------------+ - | open | True/False value indicating whether the item's children should| - | | be displayed or hidden. | + | open | ``True``/``False`` value indicating whether the item's | + | | children should be displayed or hidden. | +--------+---------------------------------------------------------------+ | tags | A list of tags associated with this item. | +--------+---------------------------------------------------------------+ @@ -997,7 +997,7 @@ ttk.Treeview The minimum width of the column in pixels. The treeview widget will not make the column any smaller than specified by this option when the widget is resized or the user drags a column. - * stretch: True/False + * stretch: ``True``/``False`` Specifies whether the column's width should be adjusted when the widget is resized. * width: width diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 5c641ef46d145e..a1aceba96ce030 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -11,7 +11,7 @@ -------------- This module provides constants which represent the numeric values of leaf nodes -of the parse tree (terminal tokens). Refer to the file :file:`Grammar/Grammar` +of the parse tree (terminal tokens). Refer to the file :file:`Grammar/Tokens` in the Python distribution for the definitions of the names in the context of the language grammar. The specific numeric values which the names map to may change between Python versions. @@ -29,17 +29,17 @@ functions. The functions mirror definitions in the Python C header files. .. function:: ISTERMINAL(x) - Return true for terminal token values. + Return ``True`` for terminal token values. .. function:: ISNONTERMINAL(x) - Return true for non-terminal token values. + Return ``True`` for non-terminal token values. .. function:: ISEOF(x) - Return true if *x* is the marker indicating the end of input. + Return ``True`` if *x* is the marker indicating the end of input. The token constants are: @@ -70,6 +70,7 @@ the :mod:`tokenize` module. .. data:: TYPE_COMMENT + :noindex: Token value indicating that a type comment was recognized. Such tokens are only produced when :func:`ast.parse()` is invoked with @@ -87,7 +88,7 @@ the :mod:`tokenize` module. now tokenized as :data:`NAME` tokens. .. versionchanged:: 3.8 - Added :data:`TYPE_COMMENT`. + Added :data:`TYPE_COMMENT`, :data:`TYPE_IGNORE`, :data:`COLONEQUAL`. Added :data:`AWAIT` and :data:`ASYNC` tokens back (they're needed to support parsing older Python versions for :func:`ast.parse` with ``feature_version`` set to 6 or lower). diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 111289c767f35c..11f569df2e7cde 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -13,7 +13,7 @@ The :mod:`tokenize` module provides a lexical scanner for Python source code, implemented in Python. The scanner in this module returns comments as tokens -as well, making it useful for implementing "pretty-printers," including +as well, making it useful for implementing "pretty-printers", including colorizers for on-screen displays. To simplify token stream handling, all :ref:`operator ` and @@ -39,8 +39,8 @@ The primary entry point is a :term:`generator`: column where the token begins in the source; a 2-tuple ``(erow, ecol)`` of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed (the last tuple item) - is the *logical* line; continuation lines are included. The 5 tuple is - returned as a :term:`named tuple` with the field names: + is the *physical* line. The 5 tuple is returned as a :term:`named tuple` + with the field names: ``type string start end line``. The returned :term:`named tuple` has an additional property named @@ -278,3 +278,22 @@ The exact token type names can be displayed using the :option:`-e` option: 4,10-4,11: RPAR ')' 4,11-4,12: NEWLINE '\n' 5,0-5,0: ENDMARKER '' + +Example of tokenizing a file programmatically, reading unicode +strings instead of bytes with :func:`generate_tokens`:: + + import tokenize + + with tokenize.open('hello.py') as f: + tokens = tokenize.generate_tokens(f.readline) + for token in tokens: + print(token) + +Or reading bytes directly with :func:`.tokenize`:: + + import tokenize + + with open('hello.py', 'rb') as f: + tokens = tokenize.tokenize(f.readline) + for token in tokens: + print(token) diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index 5cb7029adf5e9e..c2732d900bc138 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -42,6 +42,9 @@ all Python modules imported during the execution into the current directory. Display the version of the module and exit. +.. versionadded:: 3.8 + Added ``--module`` option that allows to run an executable module. + Main options ^^^^^^^^^^^^ @@ -163,7 +166,7 @@ Programmatic Interface environments. If not defined, *globals* and *locals* default to empty dictionaries. - .. method:: runfunc(func, *args, **kwds) + .. method:: runfunc(func, /, *args, **kwds) Call *func* with the given arguments under control of the :class:`Trace` object with the current tracing parameters. diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 2d327c02540995..20f668c7282028 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -202,10 +202,8 @@ ignoring ```` and ```` files:: print("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] - # replace "/path/to/module/file.py" with "module/file.py" - filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print("#%s: %s:%s: %.1f KiB" - % (index, filename, frame.lineno, stat.size / 1024)) + % (index, frame.filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line) @@ -251,6 +249,47 @@ Example of output of the Python test suite:: See :meth:`Snapshot.statistics` for more options. +Record the current and peak size of all traced memory blocks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following code computes two sums like ``0 + 1 + 2 + ...`` inefficiently, by +creating a list of those numbers. This list consumes a lot of memory +temporarily. We can use :func:`get_traced_memory` and :func:`reset_peak` to +observe the small memory usage after the sum is computed as well as the peak +memory usage during the computations:: + + import tracemalloc + + tracemalloc.start() + + # Example code: compute a sum with a large temporary list + large_sum = sum(list(range(100000))) + + first_size, first_peak = tracemalloc.get_traced_memory() + + tracemalloc.reset_peak() + + # Example code: compute a sum with a small temporary list + small_sum = sum(list(range(1000))) + + second_size, second_peak = tracemalloc.get_traced_memory() + + print(f"{first_size=}, {first_peak=}") + print(f"{second_size=}, {second_peak=}") + +Output:: + + first_size=664, first_peak=3592984 + second_size=804, second_peak=29704 + +Using :func:`reset_peak` ensured we could accurately record the peak during the +computation of ``small_sum``, even though it is much smaller than the overall +peak size of memory blocks since the :func:`start` call. Without the call to +:func:`reset_peak`, ``second_peak`` would still be the peak from the +computation ``large_sum`` (that is, equal to ``first_peak``). In this case, +both peaks are much higher than the final memory usage, and which suggests we +could optimise (by removing the unnecessary call to :class:`list`, and writing +``sum(range(...))``). API --- @@ -291,6 +330,24 @@ Functions :mod:`tracemalloc` module as a tuple: ``(current: int, peak: int)``. +.. function:: reset_peak() + + Set the peak size of memory blocks traced by the :mod:`tracemalloc` module + to the current size. + + Do nothing if the :mod:`tracemalloc` module is not tracing memory + allocations. + + This function only modifies the recorded peak size, and does not modify or + clear any traces, unlike :func:`clear_traces`. Snapshots taken with + :func:`take_snapshot` before a call to :func:`reset_peak` can be + meaningfully compared to snapshots taken after the call. + + See also :func:`get_traced_memory`. + + .. versionadded:: 3.9 + + .. function:: get_tracemalloc_memory() Get the memory usage in bytes of the :mod:`tracemalloc` module used to store @@ -313,6 +370,9 @@ Functions frames. By default, a trace of a memory block only stores the most recent frame: the limit is ``1``. *nframe* must be greater or equal to ``1``. + You can still read the original number of total frames that composed the + traceback by looking at the :attr:`Traceback.total_nframe` attribute. + Storing more than ``1`` frame is only useful to compute statistics grouped by ``'traceback'`` or to compute cumulative statistics: see the :meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods. @@ -659,6 +719,9 @@ Traceback When a snapshot is taken, tracebacks of traces are limited to :func:`get_traceback_limit` frames. See the :func:`take_snapshot` function. + The original number of frames of the traceback is stored in the + :attr:`Traceback.total_nframe` attribute. That allows to know if a traceback + has been truncated by the traceback limit. The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` instance. @@ -666,6 +729,15 @@ Traceback .. versionchanged:: 3.7 Frames are now sorted from the oldest to the most recent, instead of most recent to oldest. + .. attribute:: total_nframe + + Total number of frames that composed the traceback before truncation. + This attribute can be set to ``None`` if the information is not + available. + + .. versionchanged:: 3.9 + The :attr:`Traceback.total_nframe` attribute was added. + .. method:: format(limit=None, most_recent_first=False) Format the traceback as a list of lines with newlines. Use the diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 3d90d3cbd974f5..d3487537df99a9 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -20,8 +20,8 @@ Introduction ============ Turtle graphics is a popular way for introducing programming to kids. It was -part of the original Logo programming language developed by Wally Feurzig and -Seymour Papert in 1966. +part of the original Logo programming language developed by Wally Feurzeig, +Seymour Papert and Cynthia Solomon in 1967. Imagine a robotic turtle starting at (0, 0) in the x-y plane. After an ``import turtle``, give it the command ``turtle.forward(15)``, and it moves (on-screen!) 15 pixels in the @@ -1051,6 +1051,11 @@ Filling Fill the shape drawn after the last call to :func:`begin_fill`. + Whether or not overlap regions for self-intersecting polygons + or multiple shapes are filled depends on the operating system graphics, + type of overlap, and number of overlaps. For example, the Turtle star + above may be either all yellow or have some white regions. + .. doctest:: :skipif: _tkinter is None @@ -1064,6 +1069,7 @@ More drawing control ~~~~~~~~~~~~~~~~~~~~ .. function:: reset() + :noindex: Delete the turtle's drawings from the screen, re-center the turtle and set variables to the default values. @@ -1085,6 +1091,7 @@ More drawing control .. function:: clear() + :noindex: Delete the turtle's drawings from the screen. Do not move turtle. State and position of the turtle as well as drawings of other turtles are not affected. @@ -1357,6 +1364,7 @@ Using events ------------ .. function:: onclick(fun, btn=1, add=None) + :noindex: :param fun: a function with two arguments which will be called with the coordinates of the clicked point on the canvas diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 07c3a2e7f68276..79acdf4499afd2 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -98,6 +98,9 @@ the types that arise only incidentally during processing such as the Typical use of these names is for :func:`isinstance` or :func:`issubclass` checks. + +If you instantiate any of these types, note that signatures may vary between Python versions. + Standard names are defined for the following types: .. data:: FunctionType @@ -129,12 +132,22 @@ Standard names are defined for the following types: .. versionadded:: 3.6 -.. data:: CodeType +.. class:: CodeType(**kwargs) .. index:: builtin: compile The type for code objects such as returned by :func:`compile`. + .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags CodeType + + Note that the audited arguments may not match the names or positions + required by the initializer. + + .. method:: CodeType.replace(**kwargs) + + Return a copy of the code object with new values for the specified fields. + + .. versionadded:: 3.8 .. data:: CellType @@ -269,6 +282,11 @@ Standard names are defined for the following types: .. versionadded:: 3.3 + .. versionchanged:: 3.9 + + Updated to support the new union (``|``) operator from :pep:`584`, which + simply delegates to the underlying mapping. + .. describe:: key in proxy Return ``True`` if the underlying mapping has a key *key*, else @@ -311,6 +329,12 @@ Standard names are defined for the following types: Return a new view of the underlying mapping's values. + .. describe:: reversed(proxy) + + Return a reverse iterator over the keys of the underlying mapping. + + .. versionadded:: 3.9 + Additional Utility Classes and Functions ---------------------------------------- @@ -327,12 +351,11 @@ Additional Utility Classes and Functions The type is roughly equivalent to the following code:: class SimpleNamespace: - def __init__(self, **kwargs): + def __init__(self, /, **kwargs): self.__dict__.update(kwargs) def __repr__(self): - keys = sorted(self.__dict__) - items = ("{}={!r}".format(k, self.__dict__[k]) for k in keys) + items = (f"{k}={v!r}" for k, v in self.__dict__.items()) return "{}({})".format(type(self).__name__, ", ".join(items)) def __eq__(self, other): @@ -344,6 +367,9 @@ Additional Utility Classes and Functions .. versionadded:: 3.3 + .. versionchanged:: 3.9 + Attribute order in the repr changed from alphabetical to insertion (like + ``dict``). .. function:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None) @@ -355,7 +381,7 @@ Additional Utility Classes and Functions class's __getattr__ method; this is done by raising AttributeError. This allows one to have properties active on an instance, and have virtual - attributes on the class with the same name (see Enum for an example). + attributes on the class with the same name (see :class:`enum.Enum` for an example). .. versionadded:: 3.4 diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c2523ed5296003..d31c65d38e1abb 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1,3 +1,4 @@ +======================================== :mod:`typing` --- Support for type hints ======================================== @@ -10,14 +11,14 @@ .. note:: - The typing module has been included in the standard library on a - :term:`provisional basis `. New features might - be added and API may change even between minor releases if deemed - necessary by the core developers. + The Python runtime does not enforce function and variable type annotations. + They can be used by third party tools such as type checkers, IDEs, linters, + etc. -------------- -This module supports type hints as specified by :pep:`484` and :pep:`526`. +This module provides runtime support for type hints as specified by +:pep:`484`, :pep:`526`, :pep:`544`, :pep:`586`, :pep:`589`, and :pep:`591`. The most fundamental support consists of the types :data:`Any`, :data:`Union`, :data:`Tuple`, :data:`Callable`, :class:`TypeVar`, and :class:`Generic`. For full specification please see :pep:`484`. For @@ -34,7 +35,7 @@ In the function ``greeting``, the argument ``name`` is expected to be of type arguments. Type aliases ------------- +============ A type alias is defined by assigning the type to the alias. In this example, ``Vector`` and ``List[float]`` will be treated as interchangeable synonyms:: @@ -72,7 +73,7 @@ Note that ``None`` as a type hint is a special case and is replaced by .. _distinct: NewType -------- +======= Use the :func:`NewType` helper function to create distinct types:: @@ -149,7 +150,7 @@ See :pep:`484` for more details. .. versionadded:: 3.5.2 Callable --------- +======== Frameworks expecting callback functions of specific signatures might be type hinted using ``Callable[[Arg1Type, Arg2Type], ReturnType]``. @@ -172,7 +173,7 @@ for the list of arguments in the type hint: ``Callable[..., ReturnType]``. .. _generics: Generics --------- +======== Since type information about objects kept in containers cannot be statically inferred in a generic way, abstract base classes have been extended to support @@ -199,7 +200,7 @@ called :class:`TypeVar`. User-defined generic types --------------------------- +========================== A user-defined class can be defined as a generic class. @@ -231,8 +232,8 @@ A user-defined class can be defined as a generic class. single type parameter ``T`` . This also makes ``T`` valid as a type within the class body. -The :class:`Generic` base class uses a metaclass that defines -:meth:`__getitem__` so that ``LoggedVar[t]`` is valid as a type:: +The :class:`Generic` base class defines :meth:`__class_getitem__` so that +``LoggedVar[t]`` is valid as a type:: from typing import Iterable @@ -307,23 +308,24 @@ User defined generic type aliases are also supported. Examples:: def inproduct(v: Vec[T]) -> T: # Same as Iterable[Tuple[T, T]] return sum(x*y for x, y in v) -The metaclass used by :class:`Generic` is a subclass of :class:`abc.ABCMeta`. -A generic class can be an ABC by including abstract methods or properties, -and generic classes can also have ABCs as base classes without a metaclass +.. versionchanged:: 3.7 + :class:`Generic` no longer has a custom metaclass. + +A user-defined generic class can have ABCs as base classes without a metaclass conflict. Generic metaclasses are not supported. The outcome of parameterizing generics is cached, and most types in the typing module are hashable and comparable for equality. The :data:`Any` type --------------------- +==================== A special kind of type is :data:`Any`. A static type checker will treat every type as being compatible with :data:`Any` and :data:`Any` as being compatible with every type. This means that it is possible to perform any operation or method call on a -value of type on :data:`Any` and assign it to any variable:: +value of type :data:`Any` and assign it to any variable:: from typing import Any @@ -392,72 +394,193 @@ it as a return value) of a more specialized type is a type error. For example:: Use :class:`object` to indicate that a value could be any type in a typesafe manner. Use :data:`Any` to indicate that a value is dynamically typed. -Classes, functions, and decorators ----------------------------------- -The module defines the following classes, functions and decorators: +Nominal vs structural subtyping +=============================== -.. class:: TypeVar +Initially :pep:`484` defined Python static type system as using +*nominal subtyping*. This means that a class ``A`` is allowed where +a class ``B`` is expected if and only if ``A`` is a subclass of ``B``. - Type variable. +This requirement previously also applied to abstract base classes, such as +:class:`Iterable`. The problem with this approach is that a class had +to be explicitly marked to support them, which is unpythonic and unlike +what one would normally do in idiomatic dynamically typed Python code. +For example, this conforms to the :pep:`484`:: - Usage:: + from typing import Sized, Iterable, Iterator - T = TypeVar('T') # Can be anything - A = TypeVar('A', str, bytes) # Must be str or bytes + class Bucket(Sized, Iterable[int]): + ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function definitions. See class Generic for more - information on generic types. Generic functions work as follows:: +:pep:`544` allows to solve this problem by allowing users to write +the above code without explicit base classes in the class definition, +allowing ``Bucket`` to be implicitly considered a subtype of both ``Sized`` +and ``Iterable[int]`` by static type checkers. This is known as +*structural subtyping* (or static duck-typing):: - def repeat(x: T, n: int) -> Sequence[T]: - """Return a list containing n references to x.""" - return [x]*n + from typing import Iterator, Iterable - def longest(x: A, y: A) -> A: - """Return the longest of two strings.""" - return x if len(x) >= len(y) else y + class Bucket: # Note: no base classes + ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... - The latter example's signature is essentially the overloading - of ``(str, str) -> str`` and ``(bytes, bytes) -> bytes``. Also note - that if the arguments are instances of some subclass of :class:`str`, - the return type is still plain :class:`str`. + def collect(items: Iterable[int]) -> int: ... + result = collect(Bucket()) # Passes type check - At runtime, ``isinstance(x, T)`` will raise :exc:`TypeError`. In general, - :func:`isinstance` and :func:`issubclass` should not be used with types. +Moreover, by subclassing a special class :class:`Protocol`, a user +can define new custom protocols to fully enjoy structural subtyping +(see examples below). - Type variables may be marked covariant or contravariant by passing - ``covariant=True`` or ``contravariant=True``. See :pep:`484` for more - details. By default type variables are invariant. Alternatively, - a type variable may specify an upper bound using ``bound=``. - This means that an actual type substituted (explicitly or implicitly) - for the type variable must be a subclass of the boundary type, - see :pep:`484`. +Module contents +=============== -.. class:: Generic +The module defines the following classes, functions and decorators. - Abstract base class for generic types. +.. note:: - A generic type is typically declared by inheriting from an - instantiation of this class with one or more type variables. - For example, a generic mapping type might be defined as:: + This module defines several types that are subclasses of pre-existing + standard library classes which also extend :class:`Generic` + to support type variables inside ``[]``. + These types became redundant in Python 3.9 when the + corresponding pre-existing classes were enhanced to support ``[]``. - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. + The redundant types are deprecated as of Python 3.9 but no + deprecation warnings will be issued by the interpreter. + It is expected that type checkers will flag the deprecated types + when the checked program targets Python 3.9 or newer. - This class can then be used as follows:: + The deprecated types will be removed from the :mod:`typing` module + in the first Python version released 5 years after the release of Python 3.9.0. + See details in :pep:`585`—*Type Hinting Generics In Standard Collections*. - X = TypeVar('X') - Y = TypeVar('Y') - def lookup_name(mapping: Mapping[X, Y], key: X, default: Y) -> Y: - try: - return mapping[key] - except KeyError: - return default +Special typing primitives +------------------------- + +Special types +""""""""""""" + +These can be used as types in annotations and do not support ``[]``. + +.. data:: Any + + Special type indicating an unconstrained type. + + * Every type is compatible with :data:`Any`. + * :data:`Any` is compatible with every type. + +.. data:: NoReturn + + Special type indicating that a function never returns. + For example:: + + from typing import NoReturn + + def stop() -> NoReturn: + raise RuntimeError('no way') + + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 + +Special forms +""""""""""""" + +These can be used as types in annotations using ``[]``, each having a unique syntax. + +.. data:: Tuple + + Tuple type; ``Tuple[X, Y]`` is the type of a tuple of two items + with the first item of type X and the second of type Y. The type of + the empty tuple can be written as ``Tuple[()]``. + + Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding + to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple + of an int, a float and a string. + + To specify a variable-length tuple of homogeneous type, + use literal ellipsis, e.g. ``Tuple[int, ...]``. A plain :data:`Tuple` + is equivalent to ``Tuple[Any, ...]``, and in turn to :class:`tuple`. + + .. deprecated:: 3.9 + :class:`builtins.tuple ` now supports ``[]``. See :pep:`585`. + +.. data:: Union + + Union type; ``Union[X, Y]`` means either X or Y. + + To define a union, use e.g. ``Union[int, str]``. Details: + + * The arguments must be types and there must be at least one. + + * Unions of unions are flattened, e.g.:: + + Union[Union[int, str], float] == Union[int, str, float] + + * Unions of a single argument vanish, e.g.:: + + Union[int] == int # The constructor actually returns int + + * Redundant arguments are skipped, e.g.:: + + Union[int, str, int] == Union[int, str] + + * When comparing unions, the argument order is ignored, e.g.:: + + Union[int, str] == Union[str, int] + + * You cannot subclass or instantiate a union. + + * You cannot write ``Union[X][Y]``. + + * You can use ``Optional[X]`` as a shorthand for ``Union[X, None]``. + + .. versionchanged:: 3.7 + Don't remove explicit subclasses from unions at runtime. + +.. data:: Optional + + Optional type. + + ``Optional[X]`` is equivalent to ``Union[X, None]``. + + Note that this is not the same concept as an optional argument, + which is one that has a default. An optional argument with a + default does not require the ``Optional`` qualifier on its type + annotation just because it is optional. For example:: + + def foo(arg: int = 0) -> None: + ... + + On the other hand, if an explicit value of ``None`` is allowed, the + use of ``Optional`` is appropriate, whether the argument is optional + or not. For example:: + + def foo(arg: Optional[int] = None) -> None: + ... + +.. data:: Callable + + Callable type; ``Callable[[int], str]`` is a function of (int) -> str. + + The subscription syntax must always be used with exactly two + values: the argument list and the return type. The argument list + must be a list of types or an ellipsis; the return type must be + a single type. + + There is no syntax to indicate optional or keyword arguments; + such function types are rarely used as callback types. + ``Callable[..., ReturnType]`` (literal ellipsis) can be used to + type hint a callable taking any number of arguments and returning + ``ReturnType``. A plain :data:`Callable` is equivalent to + ``Callable[..., Any]``, and in turn to + :class:`collections.abc.Callable`. + + .. deprecated:: 3.9 + :class:`collections.abc.Callable` now supports ``[]``. See :pep:`585`. .. class:: Type(Generic[CT_co]) @@ -501,185 +624,426 @@ The module defines the following classes, functions and decorators: .. versionadded:: 3.5.2 -.. class:: Iterable(Generic[T_co]) + .. deprecated:: 3.9 + :class:`builtins.type ` now supports ``[]``. See :pep:`585`. - A generic version of :class:`collections.abc.Iterable`. +.. data:: Literal -.. class:: Iterator(Iterable[T_co]) + A type that can be used to indicate to type checkers that the + corresponding variable or function parameter has a value equivalent to + the provided literal (or one of several literals). For example:: - A generic version of :class:`collections.abc.Iterator`. + def validate_simple(data: Any) -> Literal[True]: # always returns True + ... -.. class:: Reversible(Iterable[T_co]) + MODE = Literal['r', 'rb', 'w', 'wb'] + def open_helper(file: str, mode: MODE) -> str: + ... - A generic version of :class:`collections.abc.Reversible`. + open_helper('/some/path', 'r') # Passes type check + open_helper('/other/path', 'typo') # Error in type checker -.. class:: SupportsInt + ``Literal[...]`` cannot be subclassed. At runtime, an arbitrary value + is allowed as type argument to ``Literal[...]``, but type checkers may + impose restrictions. See :pep:`586` for more details about literal types. - An ABC with one abstract method ``__int__``. + .. versionadded:: 3.8 -.. class:: SupportsFloat +.. data:: ClassVar - An ABC with one abstract method ``__float__``. + Special type construct to mark class variables. -.. class:: SupportsComplex + As introduced in :pep:`526`, a variable annotation wrapped in ClassVar + indicates that a given attribute is intended to be used as a class variable + and should not be set on instances of that class. Usage:: - An ABC with one abstract method ``__complex__``. + class Starship: + stats: ClassVar[Dict[str, int]] = {} # class variable + damage: int = 10 # instance variable -.. class:: SupportsBytes + :data:`ClassVar` accepts only types and cannot be further subscribed. - An ABC with one abstract method ``__bytes__``. + :data:`ClassVar` is not a class itself, and should not + be used with :func:`isinstance` or :func:`issubclass`. + :data:`ClassVar` does not change Python runtime behavior, but + it can be used by third-party type checkers. For example, a type checker + might flag the following code as an error:: -.. class:: SupportsAbs + enterprise_d = Starship(3000) + enterprise_d.stats = {} # Error, setting class variable on instance + Starship.stats = {} # This is OK - An ABC with one abstract method ``__abs__`` that is covariant - in its return type. + .. versionadded:: 3.5.3 -.. class:: SupportsRound +.. data:: Final - An ABC with one abstract method ``__round__`` - that is covariant in its return type. + A special typing construct to indicate to type checkers that a name + cannot be re-assigned or overridden in a subclass. For example:: -.. class:: Container(Generic[T_co]) + MAX_SIZE: Final = 9000 + MAX_SIZE += 1 # Error reported by type checker - A generic version of :class:`collections.abc.Container`. + class Connection: + TIMEOUT: Final[int] = 10 -.. class:: Hashable + class FastConnector(Connection): + TIMEOUT = 1 # Error reported by type checker - An alias to :class:`collections.abc.Hashable` + There is no runtime checking of these properties. See :pep:`591` for + more details. -.. class:: Sized + .. versionadded:: 3.8 - An alias to :class:`collections.abc.Sized` +.. data:: Annotated -.. class:: Collection(Sized, Iterable[T_co], Container[T_co]) + A type, introduced in :pep:`593` (``Flexible function and variable + annotations``), to decorate existing types with context-specific metadata + (possibly multiple pieces of it, as ``Annotated`` is variadic). + Specifically, a type ``T`` can be annotated with metadata ``x`` via the + typehint ``Annotated[T, x]``. This metadata can be used for either static + analysis or at runtime. If a library (or tool) encounters a typehint + ``Annotated[T, x]`` and has no special logic for metadata ``x``, it + should ignore it and simply treat the type as ``T``. Unlike the + ``no_type_check`` functionality that currently exists in the ``typing`` + module which completely disables typechecking annotations on a function + or a class, the ``Annotated`` type allows for both static typechecking + of ``T`` (e.g., via mypy or Pyre, which can safely ignore ``x``) + together with runtime access to ``x`` within a specific application. - A generic version of :class:`collections.abc.Collection` + Ultimately, the responsibility of how to interpret the annotations (if + at all) is the responsibility of the tool or library encountering the + ``Annotated`` type. A tool or library encountering an ``Annotated`` type + can scan through the annotations to determine if they are of interest + (e.g., using ``isinstance()``). - .. versionadded:: 3.6 + When a tool or a library does not support annotations or encounters an + unknown annotation it should just ignore it and treat annotated type as + the underlying type. -.. class:: AbstractSet(Sized, Collection[T_co]) + It's up to the tool consuming the annotations to decide whether the + client is allowed to have several annotations on one type and how to + merge those annotations. - A generic version of :class:`collections.abc.Set`. + Since the ``Annotated`` type allows you to put several annotations of + the same (or different) type(s) on any node, the tools or libraries + consuming those annotations are in charge of dealing with potential + duplicates. For example, if you are doing value range analysis you might + allow this:: -.. class:: MutableSet(AbstractSet[T]) + T1 = Annotated[int, ValueRange(-10, 5)] + T2 = Annotated[T1, ValueRange(-20, 3)] - A generic version of :class:`collections.abc.MutableSet`. + Passing ``include_extras=True`` to :func:`get_type_hints` lets one + access the extra annotations at runtime. -.. class:: Mapping(Sized, Collection[KT], Generic[VT_co]) + The details of the syntax: - A generic version of :class:`collections.abc.Mapping`. - This type can be used as follows:: + * The first argument to ``Annotated`` must be a valid type - def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: - return word_list[word] + * Multiple type annotations are supported (``Annotated`` supports variadic + arguments):: -.. class:: MutableMapping(Mapping[KT, VT]) + Annotated[int, ValueRange(3, 10), ctype("char")] - A generic version of :class:`collections.abc.MutableMapping`. + * ``Annotated`` must be called with at least two arguments ( + ``Annotated[int]`` is not valid) -.. class:: Sequence(Reversible[T_co], Collection[T_co]) + * The order of the annotations is preserved and matters for equality + checks:: - A generic version of :class:`collections.abc.Sequence`. + Annotated[int, ValueRange(3, 10), ctype("char")] != Annotated[ + int, ctype("char"), ValueRange(3, 10) + ] -.. class:: MutableSequence(Sequence[T]) + * Nested ``Annotated`` types are flattened, with metadata ordered + starting with the innermost annotation:: - A generic version of :class:`collections.abc.MutableSequence`. + Annotated[Annotated[int, ValueRange(3, 10)], ctype("char")] == Annotated[ + int, ValueRange(3, 10), ctype("char") + ] -.. class:: ByteString(Sequence[int]) + * Duplicated annotations are not removed:: - A generic version of :class:`collections.abc.ByteString`. + Annotated[int, ValueRange(3, 10)] != Annotated[ + int, ValueRange(3, 10), ValueRange(3, 10) + ] - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview`. + * ``Annotated`` can be used with nested and generic aliases:: - As a shorthand for this type, :class:`bytes` can be used to - annotate arguments of any of the types mentioned above. + T = TypeVar('T') + Vec = Annotated[List[Tuple[T, T]], MaxLen(10)] + V = Vec[int] -.. class:: Deque(deque, MutableSequence[T]) + V == Annotated[List[Tuple[int, int]], MaxLen(10)] - A generic version of :class:`collections.deque`. + .. versionadded:: 3.9 - .. versionadded:: 3.6.1 +Building generic types +"""""""""""""""""""""" -.. class:: List(list, MutableSequence[T]) +These are not used in annotations. They are building blocks for creating generic types. - Generic version of :class:`list`. - Useful for annotating return types. To annotate arguments it is preferred - to use an abstract collection type such as :class:`Sequence` or - :class:`Iterable`. +.. class:: Generic - This type may be used as follows:: + Abstract base class for generic types. - T = TypeVar('T', int, float) + A generic type is typically declared by inheriting from an + instantiation of this class with one or more type variables. + For example, a generic mapping type might be defined as:: - def vec2(x: T, y: T) -> List[T]: - return [x, y] + class Mapping(Generic[KT, VT]): + def __getitem__(self, key: KT) -> VT: + ... + # Etc. - def keep_positives(vector: Sequence[T]) -> List[T]: - return [item for item in vector if item > 0] + This class can then be used as follows:: -.. class:: Set(set, MutableSet[T]) + X = TypeVar('X') + Y = TypeVar('Y') - A generic version of :class:`builtins.set `. - Useful for annotating return types. To annotate arguments it is preferred - to use an abstract collection type such as :class:`AbstractSet`. + def lookup_name(mapping: Mapping[X, Y], key: X, default: Y) -> Y: + try: + return mapping[key] + except KeyError: + return default -.. class:: FrozenSet(frozenset, AbstractSet[T_co]) +.. class:: TypeVar - A generic version of :class:`builtins.frozenset `. + Type variable. -.. class:: MappingView(Sized, Iterable[T_co]) + Usage:: - A generic version of :class:`collections.abc.MappingView`. + T = TypeVar('T') # Can be anything + A = TypeVar('A', str, bytes) # Must be str or bytes -.. class:: KeysView(MappingView[KT_co], AbstractSet[KT_co]) + Type variables exist primarily for the benefit of static type + checkers. They serve as the parameters for generic types as well + as for generic function definitions. See :class:`Generic` for more + information on generic types. Generic functions work as follows:: - A generic version of :class:`collections.abc.KeysView`. + def repeat(x: T, n: int) -> Sequence[T]: + """Return a list containing n references to x.""" + return [x]*n -.. class:: ItemsView(MappingView, Generic[KT_co, VT_co]) + def longest(x: A, y: A) -> A: + """Return the longest of two strings.""" + return x if len(x) >= len(y) else y - A generic version of :class:`collections.abc.ItemsView`. + The latter example's signature is essentially the overloading + of ``(str, str) -> str`` and ``(bytes, bytes) -> bytes``. Also note + that if the arguments are instances of some subclass of :class:`str`, + the return type is still plain :class:`str`. -.. class:: ValuesView(MappingView[VT_co]) + At runtime, ``isinstance(x, T)`` will raise :exc:`TypeError`. In general, + :func:`isinstance` and :func:`issubclass` should not be used with types. - A generic version of :class:`collections.abc.ValuesView`. + Type variables may be marked covariant or contravariant by passing + ``covariant=True`` or ``contravariant=True``. See :pep:`484` for more + details. By default type variables are invariant. Alternatively, + a type variable may specify an upper bound using ``bound=``. + This means that an actual type substituted (explicitly or implicitly) + for the type variable must be a subclass of the boundary type, + see :pep:`484`. -.. class:: Awaitable(Generic[T_co]) +.. data:: AnyStr - A generic version of :class:`collections.abc.Awaitable`. + ``AnyStr`` is a type variable defined as + ``AnyStr = TypeVar('AnyStr', str, bytes)``. -.. class:: Coroutine(Awaitable[V_co], Generic[T_co T_contra, V_co]) + It is meant to be used for functions that may accept any kind of string + without allowing different kinds of strings to mix. For example:: - A generic version of :class:`collections.abc.Coroutine`. - The variance and order of type variables - correspond to those of :class:`Generator`, for example:: + def concat(a: AnyStr, b: AnyStr) -> AnyStr: + return a + b - from typing import List, Coroutine - c = None # type: Coroutine[List[str], str, int] - ... - x = c.send('hi') # type: List[str] - async def bar() -> None: - x = await c # type: int + concat(u"foo", u"bar") # Ok, output has type 'unicode' + concat(b"foo", b"bar") # Ok, output has type 'bytes' + concat(u"foo", b"bar") # Error, cannot mix unicode and bytes -.. class:: AsyncIterable(Generic[T_co]) +.. class:: Protocol(Generic) - A generic version of :class:`collections.abc.AsyncIterable`. + Base class for protocol classes. Protocol classes are defined like this:: -.. class:: AsyncIterator(AsyncIterable[T_co]) + class Proto(Protocol): + def meth(self) -> int: + ... - A generic version of :class:`collections.abc.AsyncIterator`. + Such classes are primarily used with static type checkers that recognize + structural subtyping (static duck-typing), for example:: -.. class:: ContextManager(Generic[T_co]) + class C: + def meth(self) -> int: + return 0 - A generic version of :class:`contextlib.AbstractContextManager`. + def func(x: Proto) -> int: + return x.meth() - .. versionadded:: 3.6 + func(C()) # Passes static type check -.. class:: AsyncContextManager(Generic[T_co]) + See :pep:`544` for details. Protocol classes decorated with + :func:`runtime_checkable` (described later) act as simple-minded runtime + protocols that check only the presence of given attributes, ignoring their + type signatures. - A generic version of :class:`contextlib.AbstractAsyncContextManager`. + Protocol classes can be generic, for example:: + + class GenProto(Protocol[T]): + def meth(self) -> T: + ... + + .. versionadded:: 3.8 + +.. decorator:: runtime_checkable + + Mark a protocol class as a runtime protocol. + + Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. + This raises :exc:`TypeError` when applied to a non-protocol class. This + allows a simple-minded structural check, very similar to "one trick ponies" + in :mod:`collections.abc` such as :class:`Iterable`. For example:: + + @runtime_checkable + class Closable(Protocol): + def close(self): ... + + assert isinstance(open('/some/file'), Closable) - .. versionadded:: 3.6 + .. note:: + + :func:`runtime_checkable` will check only the presence of the required methods, + not their type signatures! For example, :class:`builtins.complex ` + implements :func:`__float__`, therefore it passes an :func:`issubclass` check + against :class:`SupportsFloat`. However, the ``complex.__float__`` method + exists only to raise a :class:`TypeError` with a more informative message. + + .. versionadded:: 3.8 + +Other special directives +"""""""""""""""""""""""" + +These are not used in annotations. They are building blocks for declaring types. + +.. class:: NamedTuple + + Typed version of :func:`collections.namedtuple`. + + Usage:: + + class Employee(NamedTuple): + name: str + id: int + + This is equivalent to:: + + Employee = collections.namedtuple('Employee', ['name', 'id']) + + To give a field a default value, you can assign to it in the class body:: + + class Employee(NamedTuple): + name: str + id: int = 3 + + employee = Employee('Guido') + assert employee.id == 3 + + Fields with a default value must come after any fields without a default. + + The resulting class has an extra attribute ``__annotations__`` giving a + dict that maps the field names to the field types. (The field names are in + the ``_fields`` attribute and the default values are in the + ``_field_defaults`` attribute both of which are part of the namedtuple + API.) + + ``NamedTuple`` subclasses can also have docstrings and methods:: + + class Employee(NamedTuple): + """Represents an employee.""" + name: str + id: int = 3 + + def __repr__(self) -> str: + return f'' + + Backward-compatible usage:: + + Employee = NamedTuple('Employee', [('name', str), ('id', int)]) + + .. versionchanged:: 3.6 + Added support for :pep:`526` variable annotation syntax. + + .. versionchanged:: 3.6.1 + Added support for default values, methods, and docstrings. + + .. versionchanged:: 3.8 + The ``_field_types`` and ``__annotations__`` attributes are + now regular dictionaries instead of instances of ``OrderedDict``. + + .. versionchanged:: 3.9 + Removed the ``_field_types`` attribute in favor of the more + standard ``__annotations__`` attribute which has the same information. + +.. function:: NewType(name, tp) + + A helper function to indicate a distinct type to a typechecker, + see :ref:`distinct`. At runtime it returns a function that returns + its argument. Usage:: + + UserId = NewType('UserId', int) + first_user = UserId(1) + + .. versionadded:: 3.5.2 + +.. class:: TypedDict(dict) + + Special construct to add type hints to a dictionary. + At runtime it is a plain :class:`dict`. + + ``TypedDict`` declares a dictionary type that expects all of its + instances to have a certain set of keys, where each key is + associated with a value of a consistent type. This expectation + is not checked at runtime but is only enforced by type checkers. + Usage:: + + class Point2D(TypedDict): + x: int + y: int + label: str + + a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK + b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + + assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + + The type info for introspection can be accessed via ``Point2D.__annotations__`` + and ``Point2D.__total__``. To allow using this feature with older versions + of Python that do not support :pep:`526`, ``TypedDict`` supports two additional + equivalent syntactic forms:: + + Point2D = TypedDict('Point2D', x=int, y=int, label=str) + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) + + By default, all keys must be present in a TypedDict. It is possible + to override this by specifying totality. + Usage:: + + class point2D(TypedDict, total=False): + x: int + y: int + + This means that a point2D TypedDict can have any of the keys omitted. A type + checker is only expected to support a literal False or True as the value of + the total argument. True is the default, and makes all items defined in the + class body be required. + + See :pep:`589` for more examples and detailed rules of using ``TypedDict``. + + .. versionadded:: 3.8 + +Generic concrete collections +---------------------------- + +Corresponding to built-in types +""""""""""""""""""""""""""""""" .. class:: Dict(dict, MutableMapping[KT, VT]) @@ -692,30 +1056,263 @@ The module defines the following classes, functions and decorators: def count_words(text: str) -> Dict[str, int]: ... + .. deprecated:: 3.9 + :class:`builtins.dict ` now supports ``[]``. See :pep:`585`. + +.. class:: List(list, MutableSequence[T]) + + Generic version of :class:`list`. + Useful for annotating return types. To annotate arguments it is preferred + to use an abstract collection type such as :class:`Sequence` or + :class:`Iterable`. + + This type may be used as follows:: + + T = TypeVar('T', int, float) + + def vec2(x: T, y: T) -> List[T]: + return [x, y] + + def keep_positives(vector: Sequence[T]) -> List[T]: + return [item for item in vector if item > 0] + + .. deprecated:: 3.9 + :class:`builtins.list ` now supports ``[]``. See :pep:`585`. + +.. class:: Set(set, MutableSet[T]) + + A generic version of :class:`builtins.set `. + Useful for annotating return types. To annotate arguments it is preferred + to use an abstract collection type such as :class:`AbstractSet`. + + .. deprecated:: 3.9 + :class:`builtins.set ` now supports ``[]``. See :pep:`585`. + +.. class:: FrozenSet(frozenset, AbstractSet[T_co]) + + A generic version of :class:`builtins.frozenset `. + + .. deprecated:: 3.9 + :class:`builtins.frozenset ` now supports ``[]``. See :pep:`585`. + +.. note:: :data:`Tuple` is a special form. + +Corresponding to types in :mod:`collections` +"""""""""""""""""""""""""""""""""""""""""""" + .. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) A generic version of :class:`collections.defaultdict`. .. versionadded:: 3.5.2 + .. deprecated:: 3.9 + :class:`collections.defaultdict` now supports ``[]``. See :pep:`585`. + .. class:: OrderedDict(collections.OrderedDict, MutableMapping[KT, VT]) A generic version of :class:`collections.OrderedDict`. .. versionadded:: 3.7.2 + .. deprecated:: 3.9 + :class:`collections.OrderedDict` now supports ``[]``. See :pep:`585`. + +.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + + A generic version of :class:`collections.ChainMap`. + + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 + + .. deprecated:: 3.9 + :class:`collections.ChainMap` now supports ``[]``. See :pep:`585`. + .. class:: Counter(collections.Counter, Dict[T, int]) A generic version of :class:`collections.Counter`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 -.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + .. deprecated:: 3.9 + :class:`collections.Counter` now supports ``[]``. See :pep:`585`. - A generic version of :class:`collections.ChainMap`. +.. class:: Deque(deque, MutableSequence[T]) + A generic version of :class:`collections.deque`. + + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 + .. deprecated:: 3.9 + :class:`collections.deque` now supports ``[]``. See :pep:`585`. + +Other concrete types +"""""""""""""""""""" + +.. class:: IO + TextIO + BinaryIO + + Generic type ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` + and ``BinaryIO(IO[bytes])`` + represent the types of I/O streams such as returned by + :func:`open`. These types are also in the ``typing.io`` namespace. + +.. class:: Pattern + Match + + These type aliases + correspond to the return types from :func:`re.compile` and + :func:`re.match`. These types (and the corresponding functions) + are generic in ``AnyStr`` and can be made specific by writing + ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or + ``Match[bytes]``. These types are also in the ``typing.re`` namespace. + + .. deprecated:: 3.9 + Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. See :pep:`585`. + +.. class:: Text + + ``Text`` is an alias for ``str``. It is provided to supply a forward + compatible path for Python 2 code: in Python 2, ``Text`` is an alias for + ``unicode``. + + Use ``Text`` to indicate that a value must contain a unicode string in + a manner that is compatible with both Python 2 and Python 3:: + + def add_unicode_checkmark(text: Text) -> Text: + return text + u' \u2713' + + .. versionadded:: 3.5.2 + +Abstract Base Classes +--------------------- + +Corresponding to collections in :mod:`collections.abc` +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +.. class:: AbstractSet(Sized, Collection[T_co]) + + A generic version of :class:`collections.abc.Set`. + + .. deprecated:: 3.9 + :class:`collections.abc.Set` now supports ``[]``. See :pep:`585`. + +.. class:: ByteString(Sequence[int]) + + A generic version of :class:`collections.abc.ByteString`. + + This type represents the types :class:`bytes`, :class:`bytearray`, + and :class:`memoryview` of byte sequences. + + As a shorthand for this type, :class:`bytes` can be used to + annotate arguments of any of the types mentioned above. + + .. deprecated:: 3.9 + :class:`collections.abc.ByteString` now supports ``[]``. See :pep:`585`. + +.. class:: Collection(Sized, Iterable[T_co], Container[T_co]) + + A generic version of :class:`collections.abc.Collection` + + .. versionadded:: 3.6.0 + + .. deprecated:: 3.9 + :class:`collections.abc.Collection` now supports ``[]``. See :pep:`585`. + +.. class:: Container(Generic[T_co]) + + A generic version of :class:`collections.abc.Container`. + + .. deprecated:: 3.9 + :class:`collections.abc.Container` now supports ``[]``. See :pep:`585`. + +.. class:: ItemsView(MappingView, Generic[KT_co, VT_co]) + + A generic version of :class:`collections.abc.ItemsView`. + + .. deprecated:: 3.9 + :class:`collections.abc.ItemsView` now supports ``[]``. See :pep:`585`. + +.. class:: KeysView(MappingView[KT_co], AbstractSet[KT_co]) + + A generic version of :class:`collections.abc.KeysView`. + + .. deprecated:: 3.9 + :class:`collections.abc.KeysView` now supports ``[]``. See :pep:`585`. + +.. class:: Mapping(Sized, Collection[KT], Generic[VT_co]) + + A generic version of :class:`collections.abc.Mapping`. + This type can be used as follows:: + + def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: + return word_list[word] + + .. deprecated:: 3.9 + :class:`collections.abc.Mapping` now supports ``[]``. See :pep:`585`. + +.. class:: MappingView(Sized, Iterable[T_co]) + + A generic version of :class:`collections.abc.MappingView`. + + .. deprecated:: 3.9 + :class:`collections.abc.MappingView` now supports ``[]``. See :pep:`585`. + +.. class:: MutableMapping(Mapping[KT, VT]) + + A generic version of :class:`collections.abc.MutableMapping`. + + .. deprecated:: 3.9 + :class:`collections.abc.MutableMapping` now supports ``[]``. See :pep:`585`. + +.. class:: MutableSequence(Sequence[T]) + + A generic version of :class:`collections.abc.MutableSequence`. + + .. deprecated:: 3.9 + :class:`collections.abc.MutableSequence` now supports ``[]``. See :pep:`585`. + +.. class:: MutableSet(AbstractSet[T]) + + A generic version of :class:`collections.abc.MutableSet`. + + .. deprecated:: 3.9 + :class:`collections.abc.MutableSet` now supports ``[]``. See :pep:`585`. + +.. class:: Sequence(Reversible[T_co], Collection[T_co]) + + A generic version of :class:`collections.abc.Sequence`. + + .. deprecated:: 3.9 + :class:`collections.abc.Sequence` now supports ``[]``. See :pep:`585`. + +.. class:: ValuesView(MappingView[VT_co]) + + A generic version of :class:`collections.abc.ValuesView`. + + .. deprecated:: 3.9 + :class:`collections.abc.ValuesView` now supports ``[]``. See :pep:`585`. + +Corresponding to other types in :mod:`collections.abc` +"""""""""""""""""""""""""""""""""""""""""""""""""""""" + +.. class:: Iterable(Generic[T_co]) + + A generic version of :class:`collections.abc.Iterable`. + + .. deprecated:: 3.9 + :class:`collections.abc.Iterable` now supports ``[]``. See :pep:`585`. + +.. class:: Iterator(Iterable[T_co]) + + A generic version of :class:`collections.abc.Iterator`. + + .. deprecated:: 3.9 + :class:`collections.abc.Iterator` now supports ``[]``. See :pep:`585`. + .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) A generator can be annotated by the generic type @@ -747,6 +1344,45 @@ The module defines the following classes, functions and decorators: yield start start += 1 + .. deprecated:: 3.9 + :class:`collections.abc.Generator` now supports ``[]``. See :pep:`585`. + +.. class:: Hashable + + An alias to :class:`collections.abc.Hashable` + +.. class:: Reversible(Iterable[T_co]) + + A generic version of :class:`collections.abc.Reversible`. + + .. deprecated:: 3.9 + :class:`collections.abc.Reversible` now supports ``[]``. See :pep:`585`. + +.. class:: Sized + + An alias to :class:`collections.abc.Sized` + +Asynchronous programming +"""""""""""""""""""""""" + +.. class:: Coroutine(Awaitable[V_co], Generic[T_co, T_contra, V_co]) + + A generic version of :class:`collections.abc.Coroutine`. + The variance and order of type variables + correspond to those of :class:`Generator`, for example:: + + from typing import List, Coroutine + c = None # type: Coroutine[List[str], str, int] + ... + x = c.send('hi') # type: List[str] + async def bar() -> None: + x = await c # type: int + + .. versionadded:: 3.5.3 + + .. deprecated:: 3.9 + :class:`collections.abc.Coroutine` now supports ``[]``. See :pep:`585`. + .. class:: AsyncGenerator(AsyncIterator[T_co], Generic[T_co, T_contra]) An async generator can be annotated by the generic type @@ -770,119 +1406,109 @@ The module defines the following classes, functions and decorators: yield start start = await increment(start) - Alternatively, annotate your generator as having a return type of - either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: + Alternatively, annotate your generator as having a return type of + either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: + + async def infinite_stream(start: int) -> AsyncIterator[int]: + while True: + yield start + start = await increment(start) + + .. versionadded:: 3.6.1 + + .. deprecated:: 3.9 + :class:`collections.abc.AsyncGenerator` now supports ``[]``. See :pep:`585`. + +.. class:: AsyncIterable(Generic[T_co]) + + A generic version of :class:`collections.abc.AsyncIterable`. + + .. versionadded:: 3.5.2 - async def infinite_stream(start: int) -> AsyncIterator[int]: - while True: - yield start - start = await increment(start) + .. deprecated:: 3.9 + :class:`collections.abc.AsyncIterable` now supports ``[]``. See :pep:`585`. - .. versionadded:: 3.5.4 +.. class:: AsyncIterator(AsyncIterable[T_co]) -.. class:: Text + A generic version of :class:`collections.abc.AsyncIterator`. - ``Text`` is an alias for ``str``. It is provided to supply a forward - compatible path for Python 2 code: in Python 2, ``Text`` is an alias for - ``unicode``. + .. versionadded:: 3.5.2 - Use ``Text`` to indicate that a value must contain a unicode string in - a manner that is compatible with both Python 2 and Python 3:: + .. deprecated:: 3.9 + :class:`collections.abc.AsyncIterator` now supports ``[]``. See :pep:`585`. - def add_unicode_checkmark(text: Text) -> Text: - return text + u' \u2713' +.. class:: Awaitable(Generic[T_co]) + + A generic version of :class:`collections.abc.Awaitable`. .. versionadded:: 3.5.2 -.. class:: IO - TextIO - BinaryIO + .. deprecated:: 3.9 + :class:`collections.abc.Awaitable` now supports ``[]``. See :pep:`585`. - Generic type ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` - and ``BinaryIO(IO[bytes])`` - represent the types of I/O streams such as returned by - :func:`open`. -.. class:: Pattern - Match +Context manager types +""""""""""""""""""""" - These type aliases - correspond to the return types from :func:`re.compile` and - :func:`re.match`. These types (and the corresponding functions) - are generic in ``AnyStr`` and can be made specific by writing - ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or - ``Match[bytes]``. +.. class:: ContextManager(Generic[T_co]) -.. class:: NamedTuple + A generic version of :class:`contextlib.AbstractContextManager`. - Typed version of :func:`collections.namedtuple`. + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.0 - Usage:: + .. deprecated:: 3.9 + :class:`collections.contextlib.AbstractContextManager` now supports ``[]``. See :pep:`585`. - class Employee(NamedTuple): - name: str - id: int +.. class:: AsyncContextManager(Generic[T_co]) - This is equivalent to:: + A generic version of :class:`contextlib.AbstractAsyncContextManager`. - Employee = collections.namedtuple('Employee', ['name', 'id']) + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 - To give a field a default value, you can assign to it in the class body:: + .. deprecated:: 3.9 + :class:`collections.contextlib.AbstractAsyncContextManager` now supports ``[]``. See :pep:`585`. - class Employee(NamedTuple): - name: str - id: int = 3 +Protocols +--------- - employee = Employee('Guido') - assert employee.id == 3 +These protocols are decorated with :func:`runtime_checkable`. - Fields with a default value must come after any fields without a default. +.. class:: SupportsAbs - The resulting class has an extra attribute ``__annotations__`` giving a - dict that maps the field names to the field types. (The field names are in - the ``_fields`` attribute and the default values are in the - ``_field_defaults`` attribute both of which are part of the namedtuple - API.) + An ABC with one abstract method ``__abs__`` that is covariant + in its return type. - ``NamedTuple`` subclasses can also have docstrings and methods:: +.. class:: SupportsBytes - class Employee(NamedTuple): - """Represents an employee.""" - name: str - id: int = 3 + An ABC with one abstract method ``__bytes__``. - def __repr__(self) -> str: - return f'' +.. class:: SupportsComplex - Backward-compatible usage:: + An ABC with one abstract method ``__complex__``. - Employee = NamedTuple('Employee', [('name', str), ('id', int)]) +.. class:: SupportsFloat - .. versionchanged:: 3.6 - Added support for :pep:`526` variable annotation syntax. + An ABC with one abstract method ``__float__``. - .. versionchanged:: 3.6.1 - Added support for default values, methods, and docstrings. +.. class:: SupportsIndex - .. versionchanged:: 3.8 - Deprecated the ``_field_types`` attribute in favor of the more - standard ``__annotations__`` attribute which has the same information. + An ABC with one abstract method ``__index__``. - .. versionchanged:: 3.8 - The ``_field_types`` and ``__annotations__`` attributes are - now regular dictionaries instead of instances of ``OrderedDict``. + .. versionadded:: 3.8 +.. class:: SupportsInt -.. function:: NewType(typ) + An ABC with one abstract method ``__int__``. - A helper function to indicate a distinct types to a typechecker, - see :ref:`distinct`. At runtime it returns a function that returns - its argument. Usage:: +.. class:: SupportsRound - UserId = NewType('UserId', int) - first_user = UserId(1) + An ABC with one abstract method ``__round__`` + that is covariant in its return type. - .. versionadded:: 3.5.2 +Functions and decorators +------------------------ .. function:: cast(typ, val) @@ -893,19 +1519,6 @@ The module defines the following classes, functions and decorators: runtime we intentionally don't check anything (we want this to be as fast as possible). -.. function:: get_type_hints(obj[, globals[, locals]]) - - Return a dictionary containing type hints for a function, method, module - or class object. - - This is often the same as ``obj.__annotations__``. In addition, - forward references encoded as string literals are handled by evaluating - them in ``globals`` and ``locals`` namespaces. If necessary, - ``Optional[t]`` is added for function and method annotations if a default - value equal to ``None`` is set. For a class ``C``, return - a dictionary constructed by merging all the ``__annotations__`` along - ``C.__mro__`` in reverse order. - .. decorator:: overload The ``@overload`` decorator allows describing functions and methods @@ -934,6 +1547,31 @@ The module defines the following classes, functions and decorators: See :pep:`484` for details and comparison with other typing semantics. +.. decorator:: final + + A decorator to indicate to type checkers that the decorated method + cannot be overridden, and the decorated class cannot be subclassed. + For example:: + + class Base: + @final + def done(self) -> None: + ... + class Sub(Base): + def done(self) -> None: # Error reported by type checker + ... + + @final + class Leaf: + ... + class Other(Leaf): # Error reported by type checker + ... + + There is no runtime checking of these properties. See :pep:`591` for + more details. + + .. versionadded:: 3.8 + .. decorator:: no_type_check Decorator to indicate that annotations are not type hints. @@ -969,149 +1607,80 @@ The module defines the following classes, functions and decorators: Note that returning instances of private classes is not recommended. It is usually preferable to make such classes public. -.. data:: Any - - Special type indicating an unconstrained type. - - * Every type is compatible with :data:`Any`. - * :data:`Any` is compatible with every type. - -.. data:: NoReturn - - Special type indicating that a function never returns. - For example:: - - from typing import NoReturn - - def stop() -> NoReturn: - raise RuntimeError('no way') - - .. versionadded:: 3.5.4 - -.. data:: Union - - Union type; ``Union[X, Y]`` means either X or Y. - - To define a union, use e.g. ``Union[int, str]``. Details: - - * The arguments must be types and there must be at least one. - - * Unions of unions are flattened, e.g.:: - - Union[Union[int, str], float] == Union[int, str, float] - - * Unions of a single argument vanish, e.g.:: - - Union[int] == int # The constructor actually returns int - - * Redundant arguments are skipped, e.g.:: - - Union[int, str, int] == Union[int, str] - - * When comparing unions, the argument order is ignored, e.g.:: - - Union[int, str] == Union[str, int] - - * You cannot subclass or instantiate a union. - - * You cannot write ``Union[X][Y]``. - - * You can use ``Optional[X]`` as a shorthand for ``Union[X, None]``. - - .. versionchanged:: 3.7 - Don't remove explicit subclasses from unions at runtime. - -.. data:: Optional - - Optional type. - - ``Optional[X]`` is equivalent to ``Union[X, None]``. - - Note that this is not the same concept as an optional argument, - which is one that has a default. An optional argument with a - default does not require the ``Optional`` qualifier on its type - annotation just because it is optional. For example:: - - def foo(arg: int = 0) -> None: - ... - - On the other hand, if an explicit value of ``None`` is allowed, the - use of ``Optional`` is appropriate, whether the argument is optional - or not. For example:: - - def foo(arg: Optional[int] = None) -> None: - ... +Introspection helpers +--------------------- -.. data:: Tuple +.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) - Tuple type; ``Tuple[X, Y]`` is the type of a tuple of two items - with the first item of type X and the second of type Y. + Return a dictionary containing type hints for a function, method, module + or class object. - Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding - to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple - of an int, a float and a string. + This is often the same as ``obj.__annotations__``. In addition, + forward references encoded as string literals are handled by evaluating + them in ``globals`` and ``locals`` namespaces. If necessary, + ``Optional[t]`` is added for function and method annotations if a default + value equal to ``None`` is set. For a class ``C``, return + a dictionary constructed by merging all the ``__annotations__`` along + ``C.__mro__`` in reverse order. - To specify a variable-length tuple of homogeneous type, - use literal ellipsis, e.g. ``Tuple[int, ...]``. A plain :data:`Tuple` - is equivalent to ``Tuple[Any, ...]``, and in turn to :class:`tuple`. + The function recursively replaces all ``Annotated[T, ...]`` with ``T``, + unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for + more information). For example:: -.. data:: Callable + class Student(NamedTuple): + name: Annotated[str, 'some marker'] - Callable type; ``Callable[[int], str]`` is a function of (int) -> str. + get_type_hints(Student) == {'name': str} + get_type_hints(Student, include_extras=False) == {'name': str} + get_type_hints(Student, include_extras=True) == { + 'name': Annotated[str, 'some marker'] + } - The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types or an ellipsis; the return type must be - a single type. + .. versionchanged:: 3.9 + Added ``include_extras`` parameter as part of :pep:`593`. - There is no syntax to indicate optional or keyword arguments; - such function types are rarely used as callback types. - ``Callable[..., ReturnType]`` (literal ellipsis) can be used to - type hint a callable taking any number of arguments and returning - ``ReturnType``. A plain :data:`Callable` is equivalent to - ``Callable[..., Any]``, and in turn to - :class:`collections.abc.Callable`. +.. function:: get_args(tp) +.. function:: get_origin(tp) -.. data:: ClassVar + Provide basic introspection for generic types and special typing forms. - Special type construct to mark class variables. + For a typing object of the form ``X[Y, Z, ...]`` these functions return + ``X`` and ``(Y, Z, ...)``. If ``X`` is a generic alias for a builtin or + :mod:`collections` class, it gets normalized to the original class. + For unsupported objects return ``None`` and ``()`` correspondingly. + Examples:: - As introduced in :pep:`526`, a variable annotation wrapped in ClassVar - indicates that a given attribute is intended to be used as a class variable - and should not be set on instances of that class. Usage:: + assert get_origin(Dict[str, int]) is dict + assert get_args(Dict[int, str]) == (int, str) - class Starship: - stats: ClassVar[Dict[str, int]] = {} # class variable - damage: int = 10 # instance variable + assert get_origin(Union[int, str]) is Union + assert get_args(Union[int, str]) == (int, str) - :data:`ClassVar` accepts only types and cannot be further subscribed. + .. versionadded:: 3.8 - :data:`ClassVar` is not a class itself, and should not - be used with :func:`isinstance` or :func:`issubclass`. - :data:`ClassVar` does not change Python runtime behavior, but - it can be used by third-party type checkers. For example, a type checker - might flag the following code as an error:: +.. function:: is_typeddict(tp) - enterprise_d = Starship(3000) - enterprise_d.stats = {} # Error, setting class variable on instance - Starship.stats = {} # This is OK + Check if a type is a :class:`TypedDict`. - .. versionadded:: 3.5.3 + For example:: + class Film(TypedDict): + title: str + year: int -.. data:: AnyStr + is_typeddict(Film) # => True + is_typeddict(Union[list, str]) # => False - ``AnyStr`` is a type variable defined as - ``AnyStr = TypeVar('AnyStr', str, bytes)``. + .. versionadded:: 3.10 - It is meant to be used for functions that may accept any kind of string - without allowing different kinds of strings to mix. For example:: +.. class:: ForwardRef - def concat(a: AnyStr, b: AnyStr) -> AnyStr: - return a + b + A class used for internal typing representation of string forward references. + For example, ``List["SomeClass"]`` is implicitly transformed into + ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by + a user, but may be used by introspection tools. - concat(u"foo", u"bar") # Ok, output has type 'unicode' - concat(b"foo", b"bar") # Ok, output has type 'bytes' - concat(u"foo", b"bar") # Error, cannot mix unicode and bytes +Constant +-------- .. data:: TYPE_CHECKING @@ -1124,9 +1693,18 @@ The module defines the following classes, functions and decorators: def fun(arg: 'expensive_mod.SomeType') -> None: local_var: expensive_mod.AnotherType = other_fun() - Note that the first type annotation must be enclosed in quotes, making it a + The first type annotation must be enclosed in quotes, making it a "forward reference", to hide the ``expensive_mod`` reference from the interpreter runtime. Type annotations for local variables are not evaluated, so the second annotation does not need to be enclosed in quotes. + .. note:: + + If ``from __future__ import annotations`` is used in Python 3.7 or later, + annotations are not evaluated at function definition time. + Instead, the are stored as strings in ``__annotations__``, + This makes it unnecessary to use quotes around the annotation. + (see :pep:`563`). + .. versionadded:: 3.5.2 + diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index b019fa5b3bc9c2..dade3f265b56a7 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -17,12 +17,12 @@ This module provides access to the Unicode Character Database (UCD) which defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 12.0.0 -`_. +this database is compiled from the `UCD version 13.0.0 +`_. The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" -`_. It defines the +`_. It defines the following functions: @@ -175,6 +175,6 @@ Examples: .. rubric:: Footnotes -.. [#] http://www.unicode.org/Public/12.0.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/13.0.0/ucd/NameAliases.txt -.. [#] http://www.unicode.org/Public/12.0.0/ucd/NamedSequences.txt +.. [#] https://www.unicode.org/Public/13.0.0/ucd/NamedSequences.txt diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 16690f349822ec..e650bb1e23e03e 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -12,8 +12,9 @@ .. testsetup:: + import asyncio import unittest - from unittest.mock import Mock, MagicMock, patch, call, sentinel + from unittest.mock import Mock, MagicMock, AsyncMock, patch, call, sentinel class SomeClass: attribute = 'this is a doctest' @@ -276,6 +277,47 @@ function returns is what the call returns: 2 +Mocking asynchronous iterators +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since Python 3.8, ``AsyncMock`` and ``MagicMock`` have support to mock +:ref:`async-iterators` through ``__aiter__``. The :attr:`~Mock.return_value` +attribute of ``__aiter__`` can be used to set the return values to be used for +iteration. + + >>> mock = MagicMock() # AsyncMock also works here + >>> mock.__aiter__.return_value = [1, 2, 3] + >>> async def main(): + ... return [i async for i in mock] + ... + >>> asyncio.run(main()) + [1, 2, 3] + + +Mocking asynchronous context manager +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since Python 3.8, ``AsyncMock`` and ``MagicMock`` have support to mock +:ref:`async-context-managers` through ``__aenter__`` and ``__aexit__``. +By default, ``__aenter__`` and ``__aexit__`` are ``AsyncMock`` instances that +return an async function. + + >>> class AsyncContextManager: + ... async def __aenter__(self): + ... return self + ... async def __aexit__(self, exc_type, exc, tb): + ... pass + ... + >>> mock_instance = MagicMock(AsyncContextManager()) # AsyncMock also works here + >>> async def main(): + ... async with mock_instance as result: + ... pass + ... + >>> asyncio.run(main()) + >>> mock_instance.__aenter__.assert_awaited_once() + >>> mock_instance.__aexit__.assert_awaited_once() + + Creating a Mock from an Existing Object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -848,7 +890,7 @@ Here's an example implementation: >>> from copy import deepcopy >>> class CopyingMock(MagicMock): - ... def __call__(self, *args, **kwargs): + ... def __call__(self, /, *args, **kwargs): ... args = deepcopy(args) ... kwargs = deepcopy(kwargs) ... return super(CopyingMock, self).__call__(*args, **kwargs) @@ -1042,7 +1084,7 @@ that it takes arbitrary keyword arguments (``**kwargs``) which are then passed onto the mock constructor: >>> class Subclass(MagicMock): - ... def _get_child_mock(self, **kwargs): + ... def _get_child_mock(self, /, **kwargs): ... return MagicMock(**kwargs) ... >>> mymock = Subclass() diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index ed00ee6d0c2d83..c5360f91f518d6 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -201,9 +201,11 @@ The Mock Class .. testsetup:: + import asyncio + import inspect import unittest from unittest.mock import sentinel, DEFAULT, ANY - from unittest.mock import patch, call, Mock, MagicMock, PropertyMock + from unittest.mock import patch, call, Mock, MagicMock, PropertyMock, AsyncMock from unittest.mock import mock_open :class:`Mock` is a flexible mock object intended to replace the use of stubs and @@ -284,7 +286,7 @@ the *new_callable* argument to :func:`patch`. used to set attributes on the mock after it is created. See the :meth:`configure_mock` method for details. - .. method:: assert_called(*args, **kwargs) + .. method:: assert_called() Assert that the mock was called at least once. @@ -295,7 +297,7 @@ the *new_callable* argument to :func:`patch`. .. versionadded:: 3.6 - .. method:: assert_called_once(*args, **kwargs) + .. method:: assert_called_once() Assert that the mock was called exactly once. @@ -315,8 +317,8 @@ the *new_callable* argument to :func:`patch`. .. method:: assert_called_with(*args, **kwargs) - This method is a convenient way of asserting that calls are made in a - particular way: + This method is a convenient way of asserting that the last call has been + made in a particular way: >>> mock = Mock() >>> mock.method(1, 2, 3, test='wow') @@ -358,7 +360,7 @@ the *new_callable* argument to :func:`patch`. assert the mock has been called with the specified calls. The :attr:`mock_calls` list is checked for the calls. - If *any_order* is false (the default) then the calls must be + If *any_order* is false then the calls must be sequential. There can be extra calls before or after the specified calls. @@ -512,7 +514,6 @@ the *new_callable* argument to :func:`patch`. >>> mock.call_count 2 - .. attribute:: return_value Set this to configure the value returned by calling the mock: @@ -646,6 +647,9 @@ the *new_callable* argument to :func:`patch`. arguments and make more complex assertions. See :ref:`calls as tuples `. + .. versionchanged:: 3.8 + Added ``args`` and ``kwargs`` properties. + .. attribute:: call_args_list @@ -851,6 +855,248 @@ object:: >>> p.assert_called_once_with() +.. class:: AsyncMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs) + + An asynchronous version of :class:`MagicMock`. The :class:`AsyncMock` object will + behave so the object is recognized as an async function, and the result of a + call is an awaitable. + + >>> mock = AsyncMock() + >>> asyncio.iscoroutinefunction(mock) + True + >>> inspect.isawaitable(mock()) # doctest: +SKIP + True + + The result of ``mock()`` is an async function which will have the outcome + of ``side_effect`` or ``return_value`` after it has been awaited: + + - if ``side_effect`` is a function, the async function will return the + result of that function, + - if ``side_effect`` is an exception, the async function will raise the + exception, + - if ``side_effect`` is an iterable, the async function will return the + next value of the iterable, however, if the sequence of result is + exhausted, ``StopAsyncIteration`` is raised immediately, + - if ``side_effect`` is not defined, the async function will return the + value defined by ``return_value``, hence, by default, the async function + returns a new :class:`AsyncMock` object. + + + Setting the *spec* of a :class:`Mock` or :class:`MagicMock` to an async function + will result in a coroutine object being returned after calling. + + >>> async def async_func(): pass + ... + >>> mock = MagicMock(async_func) + >>> mock + + >>> mock() # doctest: +SKIP + + + + Setting the *spec* of a :class:`Mock`, :class:`MagicMock`, or :class:`AsyncMock` + to a class with asynchronous and synchronous functions will automatically + detect the synchronous functions and set them as :class:`MagicMock` (if the + parent mock is :class:`AsyncMock` or :class:`MagicMock`) or :class:`Mock` (if + the parent mock is :class:`Mock`). All asynchronous functions will be + :class:`AsyncMock`. + + >>> class ExampleClass: + ... def sync_foo(): + ... pass + ... async def async_foo(): + ... pass + ... + >>> a_mock = AsyncMock(ExampleClass) + >>> a_mock.sync_foo + + >>> a_mock.async_foo + + >>> mock = Mock(ExampleClass) + >>> mock.sync_foo + + >>> mock.async_foo + + + .. versionadded:: 3.8 + + .. method:: assert_awaited() + + Assert that the mock was awaited at least once. Note that this is separate + from the object having been called, the ``await`` keyword must be used: + + >>> mock = AsyncMock() + >>> async def main(coroutine_mock): + ... await coroutine_mock + ... + >>> coroutine_mock = mock() + >>> mock.called + True + >>> mock.assert_awaited() + Traceback (most recent call last): + ... + AssertionError: Expected mock to have been awaited. + >>> asyncio.run(main(coroutine_mock)) + >>> mock.assert_awaited() + + .. method:: assert_awaited_once() + + Assert that the mock was awaited exactly once. + + >>> mock = AsyncMock() + >>> async def main(): + ... await mock() + ... + >>> asyncio.run(main()) + >>> mock.assert_awaited_once() + >>> asyncio.run(main()) + >>> mock.method.assert_awaited_once() + Traceback (most recent call last): + ... + AssertionError: Expected mock to have been awaited once. Awaited 2 times. + + .. method:: assert_awaited_with(*args, **kwargs) + + Assert that the last await was with the specified arguments. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> asyncio.run(main('foo', bar='bar')) + >>> mock.assert_awaited_with('foo', bar='bar') + >>> mock.assert_awaited_with('other') + Traceback (most recent call last): + ... + AssertionError: expected call not found. + Expected: mock('other') + Actual: mock('foo', bar='bar') + + .. method:: assert_awaited_once_with(*args, **kwargs) + + Assert that the mock was awaited exactly once and with the specified + arguments. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> asyncio.run(main('foo', bar='bar')) + >>> mock.assert_awaited_once_with('foo', bar='bar') + >>> asyncio.run(main('foo', bar='bar')) + >>> mock.assert_awaited_once_with('foo', bar='bar') + Traceback (most recent call last): + ... + AssertionError: Expected mock to have been awaited once. Awaited 2 times. + + .. method:: assert_any_await(*args, **kwargs) + + Assert the mock has ever been awaited with the specified arguments. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> asyncio.run(main('foo', bar='bar')) + >>> asyncio.run(main('hello')) + >>> mock.assert_any_await('foo', bar='bar') + >>> mock.assert_any_await('other') + Traceback (most recent call last): + ... + AssertionError: mock('other') await not found + + .. method:: assert_has_awaits(calls, any_order=False) + + Assert the mock has been awaited with the specified calls. + The :attr:`await_args_list` list is checked for the awaits. + + If *any_order* is false then the awaits must be + sequential. There can be extra calls before or after the + specified awaits. + + If *any_order* is true then the awaits can be in any order, but + they must all appear in :attr:`await_args_list`. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> calls = [call("foo"), call("bar")] + >>> mock.assert_has_awaits(calls) + Traceback (most recent call last): + ... + AssertionError: Awaits not found. + Expected: [call('foo'), call('bar')] + Actual: [] + >>> asyncio.run(main('foo')) + >>> asyncio.run(main('bar')) + >>> mock.assert_has_awaits(calls) + + .. method:: assert_not_awaited() + + Assert that the mock was never awaited. + + >>> mock = AsyncMock() + >>> mock.assert_not_awaited() + + .. method:: reset_mock(*args, **kwargs) + + See :func:`Mock.reset_mock`. Also sets :attr:`await_count` to 0, + :attr:`await_args` to None, and clears the :attr:`await_args_list`. + + .. attribute:: await_count + + An integer keeping track of how many times the mock object has been awaited. + + >>> mock = AsyncMock() + >>> async def main(): + ... await mock() + ... + >>> asyncio.run(main()) + >>> mock.await_count + 1 + >>> asyncio.run(main()) + >>> mock.await_count + 2 + + .. attribute:: await_args + + This is either ``None`` (if the mock hasn’t been awaited), or the arguments that + the mock was last awaited with. Functions the same as :attr:`Mock.call_args`. + + >>> mock = AsyncMock() + >>> async def main(*args): + ... await mock(*args) + ... + >>> mock.await_args + >>> asyncio.run(main('foo')) + >>> mock.await_args + call('foo') + >>> asyncio.run(main('bar')) + >>> mock.await_args + call('bar') + + + .. attribute:: await_args_list + + This is a list of all the awaits made to the mock object in sequence (so the + length of the list is the number of times it has been awaited). Before any + awaits have been made it is an empty list. + + >>> mock = AsyncMock() + >>> async def main(*args): + ... await mock(*args) + ... + >>> mock.await_args_list + [] + >>> asyncio.run(main('foo')) + >>> mock.await_args_list + [call('foo')] + >>> asyncio.run(main('bar')) + >>> mock.await_args_list + [call('foo'), call('bar')] + + Calling ~~~~~~~ @@ -1084,8 +1330,7 @@ patch .. note:: - :func:`patch` is straightforward to use. The key is to do the patching in the - right namespace. See the section `where to patch`_. + The key is to do the patching in the right namespace. See the section `where to patch`_. .. function:: patch(target, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) @@ -1094,8 +1339,10 @@ patch is patched with a *new* object. When the function/with statement exits the patch is undone. - If *new* is omitted, then the target is replaced with a - :class:`MagicMock`. If :func:`patch` is used as a decorator and *new* is + If *new* is omitted, then the target is replaced with an + :class:`AsyncMock` if the patched object is an async function or + a :class:`MagicMock` otherwise. + If :func:`patch` is used as a decorator and *new* is omitted, the created mock is passed in as an extra argument to the decorated function. If :func:`patch` is used as a context manager the created mock is returned by the context manager. @@ -1113,8 +1360,8 @@ patch patch to pass in the object being mocked as the spec/spec_set object. *new_callable* allows you to specify a different class, or callable object, - that will be called to create the *new* object. By default :class:`MagicMock` is - used. + that will be called to create the *new* object. By default :class:`AsyncMock` + is used for async functions and :class:`MagicMock` for the rest. A more powerful form of *spec* is *autospec*. If you set ``autospec=True`` then the mock will be created with a spec from the object being replaced. @@ -1156,7 +1403,8 @@ patch "as"; very useful if :func:`patch` is creating a mock object for you. :func:`patch` takes arbitrary keyword arguments. These will be passed to - the :class:`Mock` (or *new_callable*) on construction. + :class:`AsyncMock` if the patched object is asynchronous, to + :class:`MagicMock` otherwise or to *new_callable* if specified. ``patch.dict(...)``, ``patch.multiple(...)`` and ``patch.object(...)`` are available for alternate use-cases. @@ -1278,6 +1526,10 @@ work as expected:: ... >>> test() +.. versionchanged:: 3.8 + + :func:`patch` now returns an :class:`AsyncMock` if the target is an async function. + patch.object ~~~~~~~~~~~~ @@ -1339,19 +1591,49 @@ patch.dict :func:`patch.dict` can also be called with arbitrary keyword arguments to set values in the dictionary. - :func:`patch.dict` can be used as a context manager, decorator or class - decorator. When used as a class decorator :func:`patch.dict` honours - ``patch.TEST_PREFIX`` for choosing which methods to wrap. + .. versionchanged:: 3.8 + + :func:`patch.dict` now returns the patched dictionary when used as a context + manager. + +:func:`patch.dict` can be used as a context manager, decorator or class +decorator: + + >>> foo = {} + >>> @patch.dict(foo, {'newkey': 'newvalue'}) + ... def test(): + ... assert foo == {'newkey': 'newvalue'} + >>> test() + >>> assert foo == {} + +When used as a class decorator :func:`patch.dict` honours +``patch.TEST_PREFIX`` (default to ``'test'``) for choosing which methods to wrap: + + >>> import os + >>> import unittest + >>> from unittest.mock import patch + >>> @patch.dict('os.environ', {'newkey': 'newvalue'}) + ... class TestSample(unittest.TestCase): + ... def test_sample(self): + ... self.assertEqual(os.environ['newkey'], 'newvalue') + +If you want to use a different prefix for your test, you can inform the +patchers of the different prefix by setting ``patch.TEST_PREFIX``. For +more details about how to change the value of see :ref:`test-prefix`. :func:`patch.dict` can be used to add members to a dictionary, or simply let a test change a dictionary, and ensure the dictionary is restored when the test ends. >>> foo = {} - >>> with patch.dict(foo, {'newkey': 'newvalue'}): + >>> with patch.dict(foo, {'newkey': 'newvalue'}) as patched_foo: ... assert foo == {'newkey': 'newvalue'} + ... assert patched_foo == {'newkey': 'newvalue'} + ... # You can add, update or delete keys of foo (or patched_foo, it's the same dict) + ... patched_foo['spam'] = 'eggs' ... >>> assert foo == {} + >>> assert patched_foo == {} >>> import os >>> with patch.dict('os.environ', {'newkey': 'newvalue'}): @@ -1551,6 +1833,8 @@ builtin :func:`ord`:: 101 +.. _test-prefix: + TEST_PREFIX ~~~~~~~~~~~ @@ -1732,7 +2016,7 @@ The full list of supported magic methods is: * Container methods: ``__getitem__``, ``__setitem__``, ``__delitem__``, ``__contains__``, ``__len__``, ``__iter__``, ``__reversed__`` and ``__missing__`` -* Context manager: ``__enter__`` and ``__exit__`` +* Context manager: ``__enter__``, ``__exit__``, ``__aenter__`` and ``__aexit__`` * Unary numeric methods: ``__neg__``, ``__pos__`` and ``__invert__`` * The numeric methods (including right hand and in-place variants): ``__add__``, ``__sub__``, ``__mul__``, ``__matmul__``, ``__div__``, ``__truediv__``, @@ -1744,10 +2028,14 @@ The full list of supported magic methods is: * Pickling: ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, ``__getnewargs__``, ``__getstate__`` and ``__setstate__`` * File system path representation: ``__fspath__`` +* Asynchronous iteration methods: ``__aiter__`` and ``__anext__`` .. versionchanged:: 3.8 Added support for :func:`os.PathLike.__fspath__`. +.. versionchanged:: 3.8 + Added support for ``__aenter__``, ``__aexit__``, ``__aiter__`` and ``__anext__``. + The following methods exist but are *not* supported as they are either in use by mock, can't be set dynamically, or can cause problems: @@ -1801,19 +2089,20 @@ to change the default. Methods and their defaults: -* ``__lt__``: NotImplemented -* ``__gt__``: NotImplemented -* ``__le__``: NotImplemented -* ``__ge__``: NotImplemented -* ``__int__``: 1 -* ``__contains__``: False -* ``__len__``: 0 -* ``__iter__``: iter([]) -* ``__exit__``: False -* ``__complex__``: 1j -* ``__float__``: 1.0 -* ``__bool__``: True -* ``__index__``: 1 +* ``__lt__``: ``NotImplemented`` +* ``__gt__``: ``NotImplemented`` +* ``__le__``: ``NotImplemented`` +* ``__ge__``: ``NotImplemented`` +* ``__int__``: ``1`` +* ``__contains__``: ``False`` +* ``__len__``: ``0`` +* ``__iter__``: ``iter([])`` +* ``__exit__``: ``False`` +* ``__aexit__``: ``False`` +* ``__complex__``: ``1j`` +* ``__float__``: ``1.0`` +* ``__bool__``: ``True`` +* ``__index__``: ``1`` * ``__hash__``: default hash for the mock * ``__str__``: default str for the mock * ``__sizeof__``: default sizeof for the mock @@ -2048,6 +2337,12 @@ See :ref:`auto-speccing` for examples of how to use auto-speccing with :func:`create_autospec` and the *autospec* argument to :func:`patch`. +.. versionchanged:: 3.8 + + :func:`create_autospec` now returns an :class:`AsyncMock` if the target is + an async function. + + ANY ~~~ diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 8ad2abd3d89a44..a52df9ee01578c 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -56,8 +56,8 @@ test runner Kent Beck's original paper on testing frameworks using the pattern shared by :mod:`unittest`. - `Nose `_ and `pytest `_ - Third-party unittest frameworks with a lighter-weight syntax for writing + `pytest `_ + Third-party unittest framework with a lighter-weight syntax for writing tests. For example, ``assert func(10) == 42``. `The Python Testing Tools Taxonomy `_ @@ -73,7 +73,7 @@ test runner for those new to unit testing. For production environments it is recommended that tests be driven by a continuous integration system such as `Buildbot `_, `Jenkins `_ - or `Hudson `_. + or `Travis-CI `_, or `AppVeyor `_. .. _unittest-minimal-example: @@ -510,7 +510,8 @@ that is broken and will fail, but shouldn't be counted as a failure on a :class:`TestResult`. Skipping a test is simply a matter of using the :func:`skip` :term:`decorator` -or one of its conditional variants. +or one of its conditional variants, calling :meth:`TestCase.skipTest` within a +:meth:`~TestCase.setUp` or test method, or raising :exc:`SkipTest` directly. Basic skipping looks like this:: @@ -531,16 +532,23 @@ Basic skipping looks like this:: # windows specific testing code pass + def test_maybe_skipped(self): + if not external_resource_available(): + self.skipTest("external resource not available") + # test code that depends on the external resource + pass + This is the output of running the example above in verbose mode:: test_format (__main__.MyTestCase) ... skipped 'not supported in this library version' test_nothing (__main__.MyTestCase) ... skipped 'demonstrating skipping' + test_maybe_skipped (__main__.MyTestCase) ... skipped 'external resource not available' test_windows_support (__main__.MyTestCase) ... skipped 'requires Windows' ---------------------------------------------------------------------- - Ran 3 tests in 0.005s + Ran 4 tests in 0.005s - OK (skipped=3) + OK (skipped=4) Classes can be skipped just like methods:: @@ -568,7 +576,7 @@ the test unless the passed object has a certain attribute:: return lambda func: func return unittest.skip("{!r} doesn't have {!r}".format(obj, attr)) -The following decorators implement test skipping and expected failures: +The following decorators and exception implement test skipping and expected failures: .. decorator:: skip(reason) @@ -902,10 +910,10 @@ Test cases .. versionadded:: 3.1 - .. method:: assertIn(first, second, msg=None) - assertNotIn(first, second, msg=None) + .. method:: assertIn(member, container, msg=None) + assertNotIn(member, container, msg=None) - Test that *first* is (or is not) in *second*. + Test that *member* is (or is not) in *container*. .. versionadded:: 3.1 @@ -942,6 +950,9 @@ Test cases | :meth:`assertLogs(logger, level) | The ``with`` block logs on *logger* | 3.4 | | ` | with minimum *level* | | +---------------------------------------------------------+--------------------------------------+------------+ + | :meth:`assertNoLogs(logger, level) | The ``with`` block does not log on | 3.10 | + | ` | *logger* with minimum *level* | | + +---------------------------------------------------------+--------------------------------------+------------+ .. method:: assertRaises(exception, callable, *args, **kwds) assertRaises(exception, *, msg=None) @@ -1113,6 +1124,24 @@ Test cases .. versionadded:: 3.4 + .. method:: assertNoLogs(logger=None, level=None) + + A context manager to test that no messages are logged on + the *logger* or one of its children, with at least the given + *level*. + + If given, *logger* should be a :class:`logging.Logger` object or a + :class:`str` giving the name of a logger. The default is the root + logger, which will catch all messages. + + If given, *level* should be either a numeric logging level or + its string equivalent (for example either ``"ERROR"`` or + :attr:`logging.ERROR`). The default is :attr:`logging.INFO`. + + Unlike :meth:`assertLogs`, nothing will be returned by the context + manager. + + .. versionadded:: 3.10 There are also other methods used to perform more specific checks, such as: @@ -1419,7 +1448,7 @@ Test cases :class:`TextTestResult` in Python 3.2. - .. method:: addCleanup(function, *args, **kwargs) + .. method:: addCleanup(function, /, *args, **kwargs) Add a function to be called after :meth:`tearDown` to cleanup resources used during the test. Functions will be called in reverse order to the @@ -1448,7 +1477,7 @@ Test cases .. versionadded:: 3.1 - .. classmethod:: addClassCleanup(function, *args, **kwargs) + .. classmethod:: addClassCleanup(function, /, *args, **kwargs) Add a function to be called after :meth:`tearDownClass` to cleanup resources used during the test class. Functions will be called in reverse @@ -1478,8 +1507,84 @@ Test cases .. versionadded:: 3.8 +.. class:: IsolatedAsyncioTestCase(methodName='runTest') + + This class provides an API similar to :class:`TestCase` and also accepts + coroutines as test functions. + + .. versionadded:: 3.8 + + .. coroutinemethod:: asyncSetUp() + + Method called to prepare the test fixture. This is called after :meth:`setUp`. + This is called immediately before calling the test method; other than + :exc:`AssertionError` or :exc:`SkipTest`, any exception raised by this method + will be considered an error rather than a test failure. The default implementation + does nothing. + .. coroutinemethod:: asyncTearDown() + + Method called immediately after the test method has been called and the + result recorded. This is called before :meth:`tearDown`. This is called even if + the test method raised an exception, so the implementation in subclasses may need + to be particularly careful about checking internal state. Any exception, other than + :exc:`AssertionError` or :exc:`SkipTest`, raised by this method will be + considered an additional error rather than a test failure (thus increasing + the total number of reported errors). This method will only be called if + the :meth:`asyncSetUp` succeeds, regardless of the outcome of the test method. + The default implementation does nothing. + + .. method:: addAsyncCleanup(function, /, *args, **kwargs) + + This method accepts a coroutine that can be used as a cleanup function. + + .. method:: run(result=None) + + Sets up a new event loop to run the test, collecting the result into + the :class:`TestResult` object passed as *result*. If *result* is + omitted or ``None``, a temporary result object is created (by calling + the :meth:`defaultTestResult` method) and used. The result object is + returned to :meth:`run`'s caller. At the end of the test all the tasks + in the event loop are cancelled. + + + An example illustrating the order:: + + from unittest import IsolatedAsyncioTestCase + + events = [] + + + class Test(IsolatedAsyncioTestCase): + + + def setUp(self): + events.append("setUp") + + async def asyncSetUp(self): + self._async_connection = await AsyncConnection() + events.append("asyncSetUp") + + async def test_response(self): + events.append("test_response") + response = await self._async_connection.get("https://example.com") + self.assertEqual(response.status_code, 200) + self.addAsyncCleanup(self.on_cleanup) + + def tearDown(self): + events.append("tearDown") + + async def asyncTearDown(self): + await self._async_connection.close() + events.append("asyncTearDown") + + async def on_cleanup(self): + events.append("cleanup") + + if __name__ == "__main__": + unittest.main() + After running the test, ``events`` would contain ``["setUp", "asyncSetUp", "test_response", "asyncTearDown", "tearDown", "cleanup"]``. .. class:: FunctionTestCase(testFunc, setUp=None, tearDown=None, description=None) @@ -2305,7 +2410,7 @@ To add cleanup code that must be run even in the case of an exception, use ``addModuleCleanup``: -.. function:: addModuleCleanup(function, *args, **kwargs) +.. function:: addModuleCleanup(function, /, *args, **kwargs) Add a function to be called after :func:`tearDownModule` to cleanup resources used during the test class. Functions will be called in reverse diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index f9936288fd42cd..f9c8ba7398f66f 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -42,8 +42,8 @@ or on combining URL components into a URL string. Parse a URL into six components, returning a 6-item :term:`named tuple`. This corresponds to the general structure of a URL: ``scheme://netloc/path;parameters?query#fragment``. - Each tuple item is a string, possibly empty. The components are not broken up in - smaller parts (for example, the network location is a single string), and % + Each tuple item is a string, possibly empty. The components are not broken up + into smaller parts (for example, the network location is a single string), and % escapes are not expanded. The delimiters as shown above are not part of the result, except for a leading slash in the *path* component, which is retained if present. For example: @@ -68,15 +68,15 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + >>> from urllib.parse import urlparse + >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') - >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', + >>> urlparse('www.cwi.nl/%7Eguido/Python.html') + ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', params='', query='', fragment='') - >>> urlparse('help/Python.html') - ParseResult(scheme='', netloc='', path='help/Python.html', params='', + >>> urlparse('help/Python.html') + ParseResult(scheme='', netloc='', path='help/Python.html', params='', query='', fragment='') The *scheme* argument gives the default addressing scheme, to be @@ -138,14 +138,14 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - >>> u - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - >>> u._replace(scheme='http') - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + >>> from urllib.parse import urlparse + >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') + >>> u + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + params='', query='', fragment='') + >>> u._replace(scheme='http') + ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + params='', query='', fragment='') .. versionchanged:: 3.2 @@ -153,7 +153,7 @@ or on combining URL components into a URL string. .. versionchanged:: 3.3 The fragment is now parsed for all URL schemes (unless *allow_fragment* is - false), in accordance with :rfc:`3986`. Previously, a whitelist of + false), in accordance with :rfc:`3986`. Previously, an allowlist of schemes that support fragments existed. .. versionchanged:: 3.6 @@ -328,22 +328,22 @@ or on combining URL components into a URL string. .. note:: - If *url* is an absolute URL (that is, starting with ``//`` or ``scheme://``), - the *url*'s host name and/or scheme will be present in the result. For example: + If *url* is an absolute URL (that is, it starts with ``//`` or ``scheme://``), + the *url*'s hostname and/or scheme will be present in the result. For example: - .. doctest:: + .. doctest:: - >>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', - ... '//www.python.org/%7Eguido') - 'http://www.python.org/%7Eguido' + >>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', + ... '//www.python.org/%7Eguido') + 'http://www.python.org/%7Eguido' - If you do not want that behavior, preprocess the *url* with :func:`urlsplit` and - :func:`urlunsplit`, removing possible *scheme* and *netloc* parts. + If you do not want that behavior, preprocess the *url* with :func:`urlsplit` and + :func:`urlunsplit`, removing possible *scheme* and *netloc* parts. .. versionchanged:: 3.5 - Behaviour updated to match the semantics defined in :rfc:`3986`. + Behavior updated to match the semantics defined in :rfc:`3986`. .. function:: urldefrag(url) @@ -370,6 +370,13 @@ or on combining URL components into a URL string. .. versionchanged:: 3.2 Result is a structured object rather than a simple 2-tuple. +.. function:: unwrap(url) + + Extract the url from a wrapped URL (that is, a string formatted as + ````, ````, ``URL:scheme://host/path`` + or ``scheme://host/path``). If *url* is not a wrapped URL, it is returned + without changes. + .. _parsing-ascii-encoded-bytes: Parsing ASCII Encoded Bytes @@ -514,15 +521,15 @@ task isn't already covered by the URL parsing functions above. Replace special characters in *string* using the ``%xx`` escape. Letters, digits, and the characters ``'_.-~'`` are never quoted. By default, this - function is intended for quoting the path section of URL. The optional *safe* - parameter specifies additional ASCII characters that should not be quoted - --- its default value is ``'/'``. + function is intended for quoting the path section of a URL. The optional + *safe* parameter specifies additional ASCII characters that should not be + quoted --- its default value is ``'/'``. - *string* may be either a :class:`str` or a :class:`bytes`. + *string* may be either a :class:`str` or a :class:`bytes` object. .. versionchanged:: 3.7 Moved from :rfc:`2396` to :rfc:`3986` for quoting URL strings. "~" is now - included in the set of reserved characters. + included in the set of unreserved characters. The optional *encoding* and *errors* parameters specify how to deal with non-ASCII characters, as accepted by the :meth:`str.encode` method. @@ -540,7 +547,7 @@ task isn't already covered by the URL parsing functions above. .. function:: quote_plus(string, safe='', encoding=None, errors=None) - Like :func:`quote`, but also replace spaces by plus signs, as required for + Like :func:`quote`, but also replace spaces with plus signs, as required for quoting HTML form values when building up a query string to go into a URL. Plus signs in the original string are escaped unless they are included in *safe*. It also does not have *safe* default to ``'/'``. @@ -559,12 +566,12 @@ task isn't already covered by the URL parsing functions above. .. function:: unquote(string, encoding='utf-8', errors='replace') - Replace ``%xx`` escapes by their single-character equivalent. + Replace ``%xx`` escapes with their single-character equivalent. The optional *encoding* and *errors* parameters specify how to decode percent-encoded sequences into Unicode characters, as accepted by the :meth:`bytes.decode` method. - *string* must be a :class:`str`. + *string* may be either a :class:`str` or a :class:`bytes` object. *encoding* defaults to ``'utf-8'``. *errors* defaults to ``'replace'``, meaning invalid sequences are replaced @@ -572,11 +579,16 @@ task isn't already covered by the URL parsing functions above. Example: ``unquote('/El%20Ni%C3%B1o/')`` yields ``'/El Niño/'``. + .. versionchanged:: 3.9 + *string* parameter supports bytes and str objects (previously only str). + + + .. function:: unquote_plus(string, encoding='utf-8', errors='replace') - Like :func:`unquote`, but also replace plus signs by spaces, as required for - unquoting HTML form values. + Like :func:`unquote`, but also replace plus signs with spaces, as required + for unquoting HTML form values. *string* must be a :class:`str`. @@ -585,10 +597,10 @@ task isn't already covered by the URL parsing functions above. .. function:: unquote_to_bytes(string) - Replace ``%xx`` escapes by their single-octet equivalent, and return a + Replace ``%xx`` escapes with their single-octet equivalent, and return a :class:`bytes` object. - *string* may be either a :class:`str` or a :class:`bytes`. + *string* may be either a :class:`str` or a :class:`bytes` object. If it is a :class:`str`, unescaped non-ASCII characters in *string* are encoded into UTF-8 bytes. @@ -619,7 +631,7 @@ task isn't already covered by the URL parsing functions above. When a sequence of two-element tuples is used as the *query* argument, the first element of each tuple is a key and the second is a value. The value element in itself can be a sequence and in that case, if - the optional parameter *doseq* is evaluates to ``True``, individual + the optional parameter *doseq* evaluates to ``True``, individual ``key=value`` pairs separated by ``'&'`` are generated for each element of the value sequence for the key. The order of parameters in the encoded string will match the order of parameter tuples in the sequence. @@ -631,11 +643,12 @@ task isn't already covered by the URL parsing functions above. To reverse this encoding process, :func:`parse_qs` and :func:`parse_qsl` are provided in this module to parse query strings into Python data structures. - Refer to :ref:`urllib examples ` to find out how urlencode - method can be used for generating query string for a URL or data for POST. + Refer to :ref:`urllib examples ` to find out how the + :func:`urllib.parse.urlencode` method can be used for generating the query + string of a URL or data for a POST request. .. versionchanged:: 3.2 - Query parameter supports bytes and string objects. + *query* supports bytes and string objects. .. versionadded:: 3.5 *quote_via* parameter. diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 14fa27bb08af20..b37f230feb6015 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -18,7 +18,7 @@ authentication, redirections, cookies and more. .. seealso:: - The `Requests package `_ + The `Requests package `_ is recommended for a higher-level HTTP client interface. @@ -55,16 +55,8 @@ The :mod:`urllib.request` module defines the following functions: The *cadefault* parameter is ignored. This function always returns an object which can work as a - :term:`context manager` and has methods such as - - * :meth:`~urllib.response.addinfourl.geturl` --- return the URL of the resource retrieved, - commonly used to determine if a redirect was followed - - * :meth:`~urllib.response.addinfourl.info` --- return the meta-information of the page, such as headers, - in the form of an :func:`email.message_from_string` instance (see - `Quick Reference to HTTP Headers `_) - - * :meth:`~urllib.response.addinfourl.getcode` -- return the HTTP status code of the response. + :term:`context manager` and has the properties *url*, *headers*, and *status*. + See :class:`urllib.response.addinfourl` for more detail on these properties. For HTTP and HTTPS URLs, this function returns a :class:`http.client.HTTPResponse` object slightly modified. In addition @@ -95,6 +87,12 @@ The :mod:`urllib.request` module defines the following functions: parameter to ``urllib.urlopen``, can be obtained by using :class:`ProxyHandler` objects. + .. audit-event:: urllib.Request fullurl,data,headers,method urllib.request.urlopen + + The default opener raises an :ref:`auditing event ` + ``urllib.Request`` with arguments ``fullurl``, ``data``, ``headers``, + ``method`` taken from the request object. + .. versionchanged:: 3.2 *cafile* and *capath* were added. @@ -118,6 +116,7 @@ The :mod:`urllib.request` module defines the following functions: :func:`ssl.create_default_context` select the system's trusted CA certificates for you. + .. function:: install_opener(opener) Install an :class:`OpenerDirector` instance as the default global opener. @@ -220,7 +219,7 @@ The following classes are provided: is not None, ``Content-Type: application/x-www-form-urlencoded`` will be added as a default. - The final two arguments are only of interest for correct handling + The next two arguments are only of interest for correct handling of third-party HTTP cookies: *origin_req_host* should be the request-host of the origin @@ -947,7 +946,7 @@ tracking URIs for which authentication credentials should always be sent. If *is_authenticated* is specified as ``True``, *realm* is ignored. -.. method:: HTTPPasswordMgr.find_user_password(realm, authuri) +.. method:: HTTPPasswordMgrWithPriorAuth.find_user_password(realm, authuri) Same as for :class:`HTTPPasswordMgrWithDefaultRealm` objects @@ -1236,7 +1235,7 @@ Here is an example of doing a ``PUT`` request using :class:`Request`:: import urllib.request DATA = b'some data' - req = urllib.request.Request(url='http://localhost:8080', data=DATA,method='PUT') + req = urllib.request.Request(url='http://localhost:8080', data=DATA, method='PUT') with urllib.request.urlopen(req) as f: pass print(f.status) @@ -1578,9 +1577,42 @@ some point in the future. :synopsis: Response classes used by urllib. The :mod:`urllib.response` module defines functions and classes which define a -minimal file like interface, including ``read()`` and ``readline()``. The -typical response object is an addinfourl instance, which defines an ``info()`` -method and that returns headers and a ``geturl()`` method that returns the url. -Functions defined by this module are used internally by the -:mod:`urllib.request` module. +minimal file-like interface, including ``read()`` and ``readline()``. +Functions defined by this module are used internally by the :mod:`urllib.request` module. +The typical response object is a :class:`urllib.response.addinfourl` instance: + +.. class:: addinfourl + + .. attribute:: url + + URL of the resource retrieved, commonly used to determine if a redirect was followed. + + .. attribute:: headers + + Returns the headers of the response in the form of an :class:`~email.message.EmailMessage` instance. + + .. attribute:: status + + .. versionadded:: 3.9 + + Status code returned by server. + + .. method:: geturl() + + .. deprecated:: 3.9 + Deprecated in favor of :attr:`~addinfourl.url`. + + .. method:: info() + + .. deprecated:: 3.9 + Deprecated in favor of :attr:`~addinfourl.headers`. + + .. attribute:: code + + .. deprecated:: 3.9 + Deprecated in favor of :attr:`~addinfourl.status`. + + .. method:: getstatus() + .. deprecated:: 3.9 + Deprecated in favor of :attr:`~addinfourl.status`. diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 412808ad44866d..5d4a36481f1dcc 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -27,7 +27,7 @@ See :pep:`405` for more information about Python virtual environments. .. seealso:: `Python Packaging User Guide: Creating and using virtual environments - `__ + `__ Creating virtual environments @@ -47,7 +47,7 @@ Creating virtual environments A virtual environment is a directory tree which contains Python executable files and other files which indicate that it is a virtual environment. - Common installation tools such as ``Setuptools`` and ``pip`` work as + Common installation tools such as setuptools_ and pip_ work as expected with virtual environments. In other words, when a virtual environment is active, they install Python packages into the virtual environment without needing to be told to do so explicitly. @@ -64,24 +64,25 @@ Creating virtual environments Python installation). When a virtual environment is active, any options that change the - installation path will be ignored from all distutils configuration files to - prevent projects being inadvertently installed outside of the virtual - environment. + installation path will be ignored from all :mod:`distutils` configuration + files to prevent projects being inadvertently installed outside of the + virtual environment. When working in a command shell, users can make a virtual environment active by running an ``activate`` script in the virtual environment's executables - directory (the precise filename is shell-dependent), which prepends the - virtual environment's directory for executables to the ``PATH`` environment - variable for the running shell. There should be no need in other - circumstances to activate a virtual environment—scripts installed into - virtual environments have a "shebang" line which points to the virtual - environment's Python interpreter. This means that the script will run with - that interpreter regardless of the value of ``PATH``. On Windows, "shebang" - line processing is supported if you have the Python Launcher for Windows - installed (this was added to Python in 3.3 - see :pep:`397` for more - details). Thus, double-clicking an installed script in a Windows Explorer - window should run the script with the correct interpreter without there - needing to be any reference to its virtual environment in ``PATH``. + directory (the precise filename and command to use the file is + shell-dependent), which prepends the virtual environment's directory for + executables to the ``PATH`` environment variable for the running shell. There + should be no need in other circumstances to activate a virtual + environment; scripts installed into virtual environments have a "shebang" + line which points to the virtual environment's Python interpreter. This means + that the script will run with that interpreter regardless of the value of + ``PATH``. On Windows, "shebang" line processing is supported if you have the + Python Launcher for Windows installed (this was added to Python in 3.3 - see + :pep:`397` for more details). Thus, double-clicking an installed script in a + Windows Explorer window should run the script with the correct interpreter + without there needing to be any reference to its virtual environment in + ``PATH``. .. _venv-api: @@ -97,7 +98,7 @@ creation according to their needs, the :class:`EnvBuilder` class. .. class:: EnvBuilder(system_site_packages=False, clear=False, \ symlinks=False, upgrade=False, with_pip=False, \ - prompt=None) + prompt=None, upgrade_deps=False) The :class:`EnvBuilder` class accepts the following keyword arguments on instantiation: @@ -121,7 +122,10 @@ creation according to their needs, the :class:`EnvBuilder` class. * ``prompt`` -- a String to be used after virtual environment is activated (defaults to ``None`` which means directory name of the environment would - be used). + be used). If the special string ``"."`` is provided, the basename of the + current directory is used as the prompt. + + * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI .. versionchanged:: 3.4 Added the ``with_pip`` parameter @@ -129,21 +133,24 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionadded:: 3.6 Added the ``prompt`` parameter + .. versionadded:: 3.9 + Added the ``upgrade_deps`` parameter + Creators of third-party virtual environment tools will be free to use the - provided ``EnvBuilder`` class as a base class. + provided :class:`EnvBuilder` class as a base class. The returned env-builder is an object which has a method, ``create``: .. method:: create(env_dir) - This method takes as required argument the path (absolute or relative to - the current directory) of the target directory which is to contain the + Create a virtual environment by specifying the target directory + (absolute or relative to the current directory) which is to contain the virtual environment. The ``create`` method will either create the environment in the specified directory, or raise an appropriate exception. - The ``create`` method of the ``EnvBuilder`` class illustrates the hooks - available for subclass customization:: + The ``create`` method of the :class:`EnvBuilder` class illustrates the + hooks available for subclass customization:: def create(self, env_dir): """ @@ -185,6 +192,14 @@ creation according to their needs, the :class:`EnvBuilder` class. Installs activation scripts appropriate to the platform into the virtual environment. + .. method:: upgrade_dependencies(context) + + Upgrades the core venv dependency packages (currently ``pip`` and + ``setuptools``) in the environment. This is done by shelling out to the + ``pip`` executable in the environment. + + .. versionadded:: 3.9 + .. method:: post_setup(context) A placeholder method which can be overridden in third party @@ -234,14 +249,23 @@ creation according to their needs, the :class:`EnvBuilder` class. There is also a module-level convenience function: .. function:: create(env_dir, system_site_packages=False, clear=False, \ - symlinks=False, with_pip=False) + symlinks=False, with_pip=False, prompt=None, \ + upgrade_deps=False) Create an :class:`EnvBuilder` with the given keyword arguments, and call its :meth:`~EnvBuilder.create` method with the *env_dir* argument. + .. versionadded:: 3.3 + .. versionchanged:: 3.4 Added the ``with_pip`` parameter + .. versionchanged:: 3.6 + Added the ``prompt`` parameter + + .. versionchanged:: 3.9 + Added the ``upgrade_deps`` parameter + An example of extending ``EnvBuilder`` -------------------------------------- @@ -262,9 +286,9 @@ subclass which installs setuptools and pip into a created virtual environment:: This builder installs setuptools and pip so that you can pip or easy_install other packages into the created virtual environment. - :param nodist: If True, setuptools and pip are not installed into the + :param nodist: If true, setuptools and pip are not installed into the created virtual environment. - :param nopip: If True, pip is not installed into the created + :param nopip: If true, pip is not installed into the created virtual environment. :param progress: If setuptools or pip are installed, the progress of the installation can be monitored by passing a progress @@ -466,3 +490,7 @@ subclass which installs setuptools and pip into a created virtual environment:: This script is also available for download `online `_. + + +.. _setuptools: https://pypi.org/project/setuptools/ +.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index d121f320d6a3eb..a481a3509d4ec8 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -19,10 +19,10 @@ Python programmers issue warnings by calling the :func:`warn` function defined in this module. (C programmers use :c:func:`PyErr_WarnEx`; see :ref:`exceptionhandling` for details). -Warning messages are normally written to ``sys.stderr``, but their disposition +Warning messages are normally written to :data:`sys.stderr`, but their disposition can be changed flexibly, from ignoring all warnings to turning them into -exceptions. The disposition of warnings can vary based on the warning category -(see below), the text of the warning message, and the source location where it +exceptions. The disposition of warnings can vary based on the :ref:`warning category +`, the text of the warning message, and the source location where it is issued. Repetitions of a particular warning for the same source location are typically suppressed. @@ -31,7 +31,7 @@ determination is made whether a message should be issued or not; next, if a message is to be issued, it is formatted and printed using a user-settable hook. The determination whether to issue a warning message is controlled by the -warning filter, which is a sequence of matching rules and actions. Rules can be +:ref:`warning filter `, which is a sequence of matching rules and actions. Rules can be added to the filter by calling :func:`filterwarnings` and reset to its default state by calling :func:`resetwarnings`. @@ -181,9 +181,9 @@ Describing Warning Filters The warnings filter is initialized by :option:`-W` options passed to the Python interpreter command line and the :envvar:`PYTHONWARNINGS` environment variable. The interpreter saves the arguments for all supplied entries without -interpretation in ``sys.warnoptions``; the :mod:`warnings` module parses these +interpretation in :data:`sys.warnoptions`; the :mod:`warnings` module parses these when it is first imported (invalid options are ignored, after printing a -message to ``sys.stderr``). +message to :data:`sys.stderr`). Individual warnings filters are specified as a sequence of fields separated by colons:: @@ -192,7 +192,7 @@ colons:: The meaning of each of these fields is as described in :ref:`warning-filter`. When listing multiple filters on a single line (as for -:envvar:`PYTHONWARNINGS`), the individual filters are separated by commas,and +:envvar:`PYTHONWARNINGS`), the individual filters are separated by commas and the filters listed later take precedence over those listed before them (as they're applied left-to-right, and the most recently applied filters take precedence over earlier ones). @@ -395,12 +395,12 @@ Available Functions .. function:: warn(message, category=None, stacklevel=1, source=None) Issue a warning, or maybe ignore it or raise an exception. The *category* - argument, if given, must be a warning category class (see above); it defaults to - :exc:`UserWarning`. Alternatively *message* can be a :exc:`Warning` instance, + argument, if given, must be a :ref:`warning category class `; it + defaults to :exc:`UserWarning`. Alternatively, *message* can be a :exc:`Warning` instance, in which case *category* will be ignored and ``message.__class__`` will be used. - In this case the message text will be ``str(message)``. This function raises an + In this case, the message text will be ``str(message)``. This function raises an exception if the particular warning issued is changed into an error by the - warnings filter see above. The *stacklevel* argument can be used by wrapper + :ref:`warnings filter `. The *stacklevel* argument can be used by wrapper functions written in Python, like this:: def deprecation(message): @@ -444,7 +444,7 @@ Available Functions Write a warning to a file. The default implementation calls ``formatwarning(message, category, filename, lineno, line)`` and writes the - resulting string to *file*, which defaults to ``sys.stderr``. You may replace + resulting string to *file*, which defaults to :data:`sys.stderr`. You may replace this function with any callable by assigning to ``warnings.showwarning``. *line* is a line of source code to be included in the warning message; if *line* is not supplied, :func:`showwarning` will diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index 60d19a8d5f7dc0..f63e0d3dce19c6 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -47,13 +47,6 @@ The :mod:`wave` module defines the following function and exception: .. versionchanged:: 3.4 Added support for unseekable files. -.. function:: openfp(file, mode) - - A synonym for :func:`.open`, maintained for backwards compatibility. - - .. deprecated-removed:: 3.7 3.9 - - .. exception:: Error An error raised when something is impossible because it violates the WAV diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 7f3d267d74c2ec..d3c3a070f38af0 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -65,8 +65,8 @@ exposed by the :mod:`weakref` module for the benefit of advanced uses. Not all objects can be weakly referenced; those objects which can include class instances, functions written in Python (but not in C), instance methods, sets, -frozensets, some :term:`file objects `, :term:`generator`\s, type -objects, sockets, arrays, deques, regular expression pattern objects, and code +frozensets, some :term:`file objects `, :term:`generators `, +type objects, sockets, arrays, deques, regular expression pattern objects, and code objects. .. versionchanged:: 3.2 @@ -80,9 +80,10 @@ support weak references but can add support through subclassing:: obj = Dict(red=1, green=2, blue=3) # this object is weak referenceable -Other built-in types such as :class:`tuple` and :class:`int` do not support weak -references even when subclassed (This is an implementation detail and may be -different across various Python implementations.). +.. impl-detail:: + + Other built-in types such as :class:`tuple` and :class:`int` do not support weak + references even when subclassed. Extension types can easily be made to support weak references; see :ref:`weakref-support`. @@ -139,6 +140,10 @@ Extension types can easily be made to support weak references; see prevent their use as dictionary keys. *callback* is the same as the parameter of the same name to the :func:`ref` function. + .. versionchanged:: 3.8 + Extended the operator support on proxy objects to include the matrix + multiplication operators ``@`` and ``@=``. + .. function:: getweakrefcount(object) @@ -158,13 +163,8 @@ Extension types can easily be made to support weak references; see application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. - .. note:: - - Caution: Because a :class:`WeakKeyDictionary` is built on top of a Python - dictionary, it must not change size when iterating over it. This can be - difficult to ensure for a :class:`WeakKeyDictionary` because actions - performed by the program during iteration may cause items in the - dictionary to vanish "by magic" (as a side effect of garbage collection). + .. versionchanged:: 3.9 + Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. :class:`WeakKeyDictionary` objects have an additional method that exposes the internal references directly. The references are not guaranteed to @@ -184,13 +184,8 @@ than needed. Mapping class that references values weakly. Entries in the dictionary will be discarded when no strong reference to the value exists any more. - .. note:: - - Caution: Because a :class:`WeakValueDictionary` is built on top of a Python - dictionary, it must not change size when iterating over it. This can be - difficult to ensure for a :class:`WeakValueDictionary` because actions performed - by the program during iteration may cause items in the dictionary to vanish "by - magic" (as a side effect of garbage collection). + .. versionchanged:: 3.9 + Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`. :class:`WeakValueDictionary` objects have an additional method that has the same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` @@ -236,7 +231,7 @@ objects. .. versionadded:: 3.4 -.. class:: finalize(obj, func, *args, **kwargs) +.. class:: finalize(obj, func, /, *args, **kwargs) Return a callable finalizer object which will be called when *obj* is garbage collected. Unlike an ordinary weak reference, a finalizer @@ -322,12 +317,6 @@ objects. types. -.. exception:: ReferenceError - - Exception raised when a proxy object is used but the underlying object has been - collected. This is the same as the standard :exc:`ReferenceError` exception. - - .. seealso:: :pep:`205` - Weak References @@ -392,7 +381,7 @@ the referent is accessed:: import weakref class ExtendedRef(weakref.ref): - def __init__(self, ob, callback=None, **annotations): + def __init__(self, ob, callback=None, /, **annotations): super(ExtendedRef, self).__init__(ob, callback) self.__counter = 0 for k, v in annotations.items(): diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 9dc5551398ee82..b7bfb655a71579 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -64,6 +64,8 @@ The following functions are defined: may work and start the operating system's associated program. However, this is neither supported nor portable. + .. audit-event:: webbrowser.open url webbrowser.open + .. function:: open_new(url) diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index e9c02610227331..dccb7db27e90cc 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -53,6 +53,8 @@ This module offers the following functions: The return value is the handle of the opened key. If the function fails, an :exc:`OSError` exception is raised. + .. audit-event:: winreg.ConnectRegistry computer_name,key winreg.ConnectRegistry + .. versionchanged:: 3.3 See :ref:`above `. @@ -75,6 +77,10 @@ This module offers the following functions: The return value is the handle of the opened key. If the function fails, an :exc:`OSError` exception is raised. + .. audit-event:: winreg.CreateKey key,sub_key,access winreg.CreateKey + + .. audit-event:: winreg.OpenKey/result key winreg.CreateKey + .. versionchanged:: 3.3 See :ref:`above `. @@ -103,6 +109,10 @@ This module offers the following functions: The return value is the handle of the opened key. If the function fails, an :exc:`OSError` exception is raised. + .. audit-event:: winreg.CreateKey key,sub_key,access winreg.CreateKeyEx + + .. audit-event:: winreg.OpenKey/result key winreg.CreateKeyEx + .. versionadded:: 3.2 .. versionchanged:: 3.3 @@ -124,6 +134,8 @@ This module offers the following functions: If the method succeeds, the entire key, including all of its values, is removed. If the method fails, an :exc:`OSError` exception is raised. + .. audit-event:: winreg.DeleteKey key,sub_key,access winreg.DeleteKey + .. versionchanged:: 3.3 See :ref:`above `. @@ -158,6 +170,8 @@ This module offers the following functions: On unsupported Windows versions, :exc:`NotImplementedError` is raised. + .. audit-event:: winreg.DeleteKey key,sub_key,access winreg.DeleteKeyEx + .. versionadded:: 3.2 .. versionchanged:: 3.3 @@ -173,6 +187,8 @@ This module offers the following functions: *value* is a string that identifies the value to remove. + .. audit-event:: winreg.DeleteValue key,value winreg.DeleteValue + .. function:: EnumKey(key, index) @@ -187,6 +203,8 @@ This module offers the following functions: typically called repeatedly until an :exc:`OSError` exception is raised, indicating, no more values are available. + .. audit-event:: winreg.EnumKey key,index winreg.EnumKey + .. versionchanged:: 3.3 See :ref:`above `. @@ -220,6 +238,8 @@ This module offers the following functions: | | :meth:`SetValueEx`) | +-------+--------------------------------------------+ + .. audit-event:: winreg.EnumValue key,index winreg.EnumValue + .. versionchanged:: 3.3 See :ref:`above `. @@ -235,6 +255,8 @@ This module offers the following functions: >>> ExpandEnvironmentStrings('%windir%') 'C:\\Windows' + .. audit-event:: winreg.ExpandEnvironmentStrings str winreg.ExpandEnvironmentStrings + .. function:: FlushKey(key) @@ -279,6 +301,8 @@ This module offers the following functions: If *key* is a handle returned by :func:`ConnectRegistry`, then the path specified in *file_name* is relative to the remote computer. + .. audit-event:: winreg.LoadKey key,sub_key,file_name winreg.LoadKey + .. function:: OpenKey(key, sub_key, reserved=0, access=KEY_READ) OpenKeyEx(key, sub_key, reserved=0, access=KEY_READ) @@ -300,6 +324,10 @@ This module offers the following functions: If the function fails, :exc:`OSError` is raised. + .. audit-event:: winreg.OpenKey key,sub_key,access winreg.OpenKey + + .. audit-event:: winreg.OpenKey/result key winreg.OpenKey + .. versionchanged:: 3.2 Allow the use of named arguments. @@ -330,6 +358,8 @@ This module offers the following functions: | | nanoseconds since Jan 1, 1601. | +-------+---------------------------------------------+ + .. audit-event:: winreg.QueryInfoKey key winreg.QueryInfoKey + .. function:: QueryValue(key, sub_key) @@ -343,10 +373,12 @@ This module offers the following functions: value set by the :func:`SetValue` method for the key identified by *key*. Values in the registry have name, type, and data components. This method - retrieves the data for a key's first value that has a NULL name. But the + retrieves the data for a key's first value that has a ``NULL`` name. But the underlying API call doesn't return the type, so always use :func:`QueryValueEx` if possible. + .. audit-event:: winreg.QueryValue key,sub_key,value_name winreg.QueryValue + .. function:: QueryValueEx(key, value_name) @@ -370,6 +402,8 @@ This module offers the following functions: | | :meth:`SetValueEx`) | +-------+-----------------------------------------+ + .. audit-event:: winreg.QueryValue key,sub_key,value_name winreg.QueryValueEx + .. function:: SaveKey(key, file_name) @@ -391,7 +425,9 @@ This module offers the following functions: `__ for more details. - This function passes NULL for *security_attributes* to the API. + This function passes ``NULL`` for *security_attributes* to the API. + + .. audit-event:: winreg.SaveKey key,file_name winreg.SaveKey .. function:: SetValue(key, sub_key, type, value) @@ -419,6 +455,8 @@ This module offers the following functions: The key identified by the *key* parameter must have been opened with :const:`KEY_SET_VALUE` access. + .. audit-event:: winreg.SetValue key,sub_key,type,value winreg.SetValue + .. function:: SetValueEx(key, value_name, reserved, type, value) @@ -447,6 +485,8 @@ This module offers the following functions: bytes) should be stored as files with the filenames stored in the configuration registry. This helps the registry perform efficiently. + .. audit-event:: winreg.SetValue key,sub_key,type,value winreg.SetValueEx + .. function:: DisableReflectionKey(key) @@ -456,13 +496,15 @@ This module offers the following functions: *key* is an already open key, or one of the predefined :ref:`HKEY_* constants `. - Will generally raise :exc:`NotImplemented` if executed on a 32-bit operating + Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating system. If the key is not on the reflection list, the function succeeds but has no effect. Disabling reflection for a key does not affect reflection of any subkeys. + .. audit-event:: winreg.DisableReflectionKey key winreg.DisableReflectionKey + .. function:: EnableReflectionKey(key) @@ -471,11 +513,13 @@ This module offers the following functions: *key* is an already open key, or one of the predefined :ref:`HKEY_* constants `. - Will generally raise :exc:`NotImplemented` if executed on a 32-bit operating + Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating system. Restoring reflection for a key does not affect reflection of any subkeys. + .. audit-event:: winreg.EnableReflectionKey key winreg.EnableReflectionKey + .. function:: QueryReflectionKey(key) @@ -486,9 +530,11 @@ This module offers the following functions: Returns ``True`` if reflection is disabled. - Will generally raise :exc:`NotImplemented` if executed on a 32-bit + Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating system. + .. audit-event:: winreg.QueryReflectionKey key winreg.QueryReflectionKey + .. _constants: @@ -741,6 +787,9 @@ integer handle, and also disconnect the Windows handle from the handle object. handle is not closed. You would call this function when you need the underlying Win32 handle to exist beyond the lifetime of the handle object. + .. audit-event:: winreg.PyHKEY.Detach key winreg.PyHKEY.Detach + + .. method:: PyHKEY.__enter__() PyHKEY.__exit__(\*exc_info) diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index ec5136742fa2c4..e92a689de0b9ba 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -144,7 +144,7 @@ also provides these miscellaneous utilities: .. function:: is_hop_by_hop(header_name) - Return true if 'header_name' is an HTTP/1.1 "Hop-by-Hop" header, as defined by + Return ``True`` if 'header_name' is an HTTP/1.1 "Hop-by-Hop" header, as defined by :rfc:`2616`. @@ -480,8 +480,8 @@ input, output, and error streams. rarely used and is not guaranteed by WSGI. On IIS<7, though, the setting can only be made on a vhost level, affecting all other script mappings, many of which break when exposed to the ``PATH_TRANSLATED`` bug. - For this reason IIS<7 is almost never deployed with the fix. (Even IIS7 - rarely uses it because there is still no UI for it.) + For this reason IIS<7 is almost never deployed with the fix (Even IIS7 + rarely uses it because there is still no UI for it.). There is no way for CGI code to tell whether the option was set, so a separate handler class is provided. It is used in the same way as @@ -767,7 +767,7 @@ This is a working "Hello World" WSGI application:: # use a function (note that you're not limited to a function, you can # use a class for example). The first argument passed to the function # is a dictionary containing CGI-style environment variables and the - # second variable is the callable object (see PEP 333). + # second variable is the callable object. def hello_world_app(environ, start_response): status = '200 OK' # HTTP Status headers = [('Content-type', 'text/plain; charset=utf-8')] # HTTP Headers diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 2423a0c15691f4..2c78cd939243a8 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -132,22 +132,31 @@ module documentation. This section lists the differences between the API and ... # Work with dom. -.. method:: Node.writexml(writer, indent="", addindent="", newl="") +.. method:: Node.writexml(writer, indent="", addindent="", newl="", + encoding=None, standalone=None) - Write XML to the writer object. The writer should have a :meth:`write` method - which matches that of the file object interface. The *indent* parameter is the - indentation of the current node. The *addindent* parameter is the incremental - indentation to use for subnodes of the current one. The *newl* parameter - specifies the string to use to terminate newlines. + Write XML to the writer object. The writer receives texts but not bytes as input, + it should have a :meth:`write` method which matches that of the file object + interface. The *indent* parameter is the indentation of the current node. + The *addindent* parameter is the incremental indentation to use for subnodes + of the current one. The *newl* parameter specifies the string to use to + terminate newlines. For the :class:`Document` node, an additional keyword argument *encoding* can be used to specify the encoding field of the XML header. + Silimarly, explicitly stating the *standalone* argument causes the + standalone document declarations to be added to the prologue of the XML + document. + If the value is set to `True`, `standalone="yes"` is added, + otherwise it is set to `"no"`. + Not stating the argument will omit the declaration from the document. + .. versionchanged:: 3.8 The :meth:`writexml` method now preserves the attribute order specified by the user. -.. method:: Node.toxml(encoding=None) +.. method:: Node.toxml(encoding=None, standalone=None) Return a string or byte string containing the XML represented by the DOM node. @@ -159,11 +168,14 @@ module documentation. This section lists the differences between the API and encoding. Encoding this string in an encoding other than UTF-8 is likely incorrect, since UTF-8 is the default encoding of XML. + The *standalone* argument behaves exactly as in :meth:`writexml`. + .. versionchanged:: 3.8 The :meth:`toxml` method now preserves the attribute order specified by the user. -.. method:: Node.toprettyxml(indent="\\t", newl="\\n", encoding=None) +.. method:: Node.toprettyxml(indent="\\t", newl="\\n", encoding=None, + standalone=None) Return a pretty-printed version of the document. *indent* specifies the indentation string and defaults to a tabulator; *newl* specifies the string @@ -172,6 +184,8 @@ module documentation. This section lists the differences between the API and The *encoding* argument behaves like the corresponding argument of :meth:`toxml`. + The *standalone* argument behaves exactly as in :meth:`writexml`. + .. versionchanged:: 3.8 The :meth:`toprettyxml` method now preserves the attribute order specified by the user. diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index 18519a75a54611..98454e18ff1f87 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -210,7 +210,7 @@ DOM Level 2 added the ability to create new :class:`Document` and .. method:: DOMImplementation.hasFeature(feature, version) - Return true if the feature identified by the pair of strings *feature* and + Return ``True`` if the feature identified by the pair of strings *feature* and *version* is implemented. @@ -335,17 +335,17 @@ All of the components of an XML document are subclasses of :class:`Node`. .. method:: Node.hasAttributes() - Returns true if the node has any attributes. + Return ``True`` if the node has any attributes. .. method:: Node.hasChildNodes() - Returns true if the node has any child nodes. + Return ``True`` if the node has any child nodes. .. method:: Node.isSameNode(other) - Returns true if *other* refers to the same node as this node. This is especially + Return ``True`` if *other* refers to the same node as this node. This is especially useful for DOM implementations which use any sort of proxy architecture (because more than one object can refer to the same node). @@ -604,12 +604,12 @@ of that class. .. method:: Element.hasAttribute(name) - Returns true if the element has an attribute named by *name*. + Return ``True`` if the element has an attribute named by *name*. .. method:: Element.hasAttributeNS(namespaceURI, localName) - Returns true if the element has an attribute named by *namespaceURI* and + Return ``True`` if the element has an attribute named by *namespaceURI* and *localName*. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 9e2c295867ca3a..7725e4d158d429 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -15,6 +15,8 @@ for parsing and creating XML data. .. versionchanged:: 3.3 This module will use a fast implementation whenever available. + +.. deprecated:: 3.3 The :mod:`xml.etree.cElementTree` module is deprecated. @@ -399,6 +401,12 @@ module. We'll be using the ``countrydata`` XML document from the # All 'neighbor' nodes that are the second child of their parent root.findall(".//neighbor[2]") +For XML with namespaces, use the usual qualified ``{namespace}tag`` notation:: + + # All dublin-core "title" tags in the document + root.findall(".//{http://purl.org/dc/elements/1.1/}title") + + Supported XPath syntax ^^^^^^^^^^^^^^^^^^^^^^ @@ -411,9 +419,16 @@ Supported XPath syntax | | For example, ``spam`` selects all child elements | | | named ``spam``, and ``spam/egg`` selects all | | | grandchildren named ``egg`` in all children named | -| | ``spam``. | +| | ``spam``. ``{namespace}*`` selects all tags in the | +| | given namespace, ``{*}spam`` selects tags named | +| | ``spam`` in any (or no) namespace, and ``{}*`` | +| | only selects tags that are not in a namespace. | +| | | +| | .. versionchanged:: 3.8 | +| | Support for star-wildcards was added. | +-----------------------+------------------------------------------------------+ -| ``*`` | Selects all child elements. For example, ``*/egg`` | +| ``*`` | Selects all child elements, including comments and | +| | processing instructions. For example, ``*/egg`` | | | selects all grandchildren named ``egg``. | +-----------------------+------------------------------------------------------+ | ``.`` | Selects the current node. This is mostly useful | @@ -465,6 +480,53 @@ Reference Functions ^^^^^^^^^ +.. function:: canonicalize(xml_data=None, *, out=None, from_file=None, **options) + + `C14N 2.0 `_ transformation function. + + Canonicalization is a way to normalise XML output in a way that allows + byte-by-byte comparisons and digital signatures. It reduced the freedom + that XML serializers have and instead generates a more constrained XML + representation. The main restrictions regard the placement of namespace + declarations, the ordering of attributes, and ignorable whitespace. + + This function takes an XML data string (*xml_data*) or a file path or + file-like object (*from_file*) as input, converts it to the canonical + form, and writes it out using the *out* file(-like) object, if provided, + or returns it as a text string if not. The output file receives text, + not bytes. It should therefore be opened in text mode with ``utf-8`` + encoding. + + Typical uses:: + + xml_data = "..." + print(canonicalize(xml_data)) + + with open("c14n_output.xml", mode='w', encoding='utf-8') as out_file: + canonicalize(xml_data, out=out_file) + + with open("c14n_output.xml", mode='w', encoding='utf-8') as out_file: + canonicalize(from_file="inputfile.xml", out=out_file) + + The configuration *options* are as follows: + + - *with_comments*: set to true to include comments (default: false) + - *strip_text*: set to true to strip whitespace before and after text content + (default: false) + - *rewrite_prefixes*: set to true to replace namespace prefixes by "n{number}" + (default: false) + - *qname_aware_tags*: a set of qname aware tag names in which prefixes + should be replaced in text content (default: empty) + - *qname_aware_attrs*: a set of qname aware attribute names in which prefixes + should be replaced in text content (default: empty) + - *exclude_attrs*: a set of attribute names that should not be serialised + - *exclude_tags*: a set of tag names that should not be serialised + + In the option list above, "a set" refers to any collection or iterable of + strings, no ordering is expected. + + .. versionadded:: 3.8 + .. function:: Comment(text=None) @@ -512,10 +574,22 @@ Functions .. versionadded:: 3.2 +.. function:: indent(tree, space=" ", level=0) + + Appends whitespace to the subtree to indent the tree visually. + This can be used to generate pretty-printed XML output. + *tree* can be an Element or ElementTree. *space* is the whitespace + string that will be inserted for each indentation level, two space + characters by default. For indenting partial subtrees inside of an + already indented tree, pass the initial indentation level as *level*. + + .. versionadded:: 3.9 + + .. function:: iselement(element) - Checks if an object appears to be a valid element object. *element* is an - element instance. Returns a true value if this is an element object. + Check if an object appears to be a valid element object. *element* is an + element instance. Return ``True`` if this is an element object. .. function:: iterparse(source, events=None, parser=None) @@ -523,8 +597,9 @@ Functions Parses an XML section into an element tree incrementally, and reports what's going on to the user. *source* is a filename or :term:`file object` containing XML data. *events* is a sequence of events to report back. The - supported events are the strings ``"start"``, ``"end"``, ``"start-ns"`` and - ``"end-ns"`` (the "ns" events are used to get detailed namespace + supported events are the strings ``"start"``, ``"end"``, ``"comment"``, + ``"pi"``, ``"start-ns"`` and ``"end-ns"`` + (the "ns" events are used to get detailed namespace information). If *events* is omitted, only ``"end"`` events are reported. *parser* is an optional parser instance. If not given, the standard :class:`XMLParser` parser is used. *parser* must be a subclass of @@ -549,6 +624,10 @@ Functions .. deprecated:: 3.4 The *parser* argument. + .. versionchanged:: 3.8 + The ``comment`` and ``pi`` events were added. + + .. function:: parse(source, parser=None) Parses an XML section into an element tree. *source* is a filename or file @@ -594,7 +673,7 @@ Functions .. function:: tostring(element, encoding="us-ascii", method="xml", *, \ - xml_declaration=None, default_namespace=None, + xml_declaration=None, default_namespace=None, \ short_empty_elements=True) Generates a string representation of an XML element, including all @@ -612,9 +691,13 @@ Functions .. versionadded:: 3.8 The *xml_declaration* and *default_namespace* parameters. + .. versionchanged:: 3.8 + The :func:`tostring` function now preserves the attribute order + specified by the user. + .. function:: tostringlist(element, encoding="us-ascii", method="xml", *, \ - xml_declaration=None, default_namespace=None, + xml_declaration=None, default_namespace=None, \ short_empty_elements=True) Generates a string representation of an XML element, including all @@ -635,6 +718,10 @@ Functions .. versionadded:: 3.8 The *xml_declaration* and *default_namespace* parameters. + .. versionchanged:: 3.8 + The :func:`tostringlist` function now preserves the attribute order + specified by the user. + .. function:: XML(text, parser=None) @@ -653,6 +740,104 @@ Functions :class:`Element` instance and a dictionary. +.. _elementtree-xinclude: + +XInclude support +---------------- + +This module provides limited support for +`XInclude directives `_, via the :mod:`xml.etree.ElementInclude` helper module. This module can be used to insert subtrees and text strings into element trees, based on information in the tree. + +Example +^^^^^^^ + +Here's an example that demonstrates use of the XInclude module. To include an XML document in the current document, use the ``{http://www.w3.org/2001/XInclude}include`` element and set the **parse** attribute to ``"xml"``, and use the **href** attribute to specify the document to include. + +.. code-block:: xml + + + + + + +By default, the **href** attribute is treated as a file name. You can use custom loaders to override this behaviour. Also note that the standard helper does not support XPointer syntax. + +To process this file, load it as usual, and pass the root element to the :mod:`xml.etree.ElementTree` module: + +.. code-block:: python + + from xml.etree import ElementTree, ElementInclude + + tree = ElementTree.parse("document.xml") + root = tree.getroot() + + ElementInclude.include(root) + +The ElementInclude module replaces the ``{http://www.w3.org/2001/XInclude}include`` element with the root element from the **source.xml** document. The result might look something like this: + +.. code-block:: xml + + + This is a paragraph. + + +If the **parse** attribute is omitted, it defaults to "xml". The href attribute is required. + +To include a text document, use the ``{http://www.w3.org/2001/XInclude}include`` element, and set the **parse** attribute to "text": + +.. code-block:: xml + + + + Copyright (c) . + + +The result might look something like: + +.. code-block:: xml + + + Copyright (c) 2003. + + +Reference +--------- + +.. _elementinclude-functions: + +Functions +^^^^^^^^^ + +.. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None) + + Default loader. This default loader reads an included resource from disk. *href* is a URL. + *parse* is for parse mode either "xml" or "text". *encoding* + is an optional text encoding. If not given, encoding is ``utf-8``. Returns the + expanded resource. If the parse mode is ``"xml"``, this is an ElementTree + instance. If the parse mode is "text", this is a Unicode string. If the + loader fails, it can return None or raise an exception. + + +.. function:: xml.etree.ElementInclude.include( elem, loader=None, base_url=None, \ + max_depth=6) + + This function expands XInclude directives. *elem* is the root element. *loader* is + an optional resource loader. If omitted, it defaults to :func:`default_loader`. + If given, it should be a callable that implements the same interface as + :func:`default_loader`. *base_url* is base URL of the original file, to resolve + relative include file references. *max_depth* is the maximum number of recursive + inclusions. Limited to reduce the risk of malicious content explosion. Pass a + negative value to disable the limitation. + + Returns the expanded resource. If the parse mode is + ``"xml"``, this is an ElementTree instance. If the parse mode is "text", + this is a Unicode string. If the loader fails, it can return None or + raise an exception. + + .. versionadded:: 3.9 + The *base_url* and *max_depth* parameters. + + .. _elementtree-element-objects: Element Objects @@ -788,18 +973,6 @@ Element Objects in the expression into the given namespace. - .. method:: getchildren() - - .. deprecated-removed:: 3.2 3.9 - Use ``list(elem)`` or iteration. - - - .. method:: getiterator(tag=None) - - .. deprecated-removed:: 3.2 3.9 - Use method :meth:`Element.iter` instead. - - .. method:: insert(index, subelement) Inserts *subelement* at the given position in this element. Raises @@ -865,6 +1038,36 @@ Element Objects if element is None: print("element not found") + Prior to Python 3.8, the serialisation order of the XML attributes of + elements was artificially made predictable by sorting the attributes by + their name. Based on the now guaranteed ordering of dicts, this arbitrary + reordering was removed in Python 3.8 to preserve the order in which + attributes were originally parsed or created by user code. + + In general, user code should try not to depend on a specific ordering of + attributes, given that the `XML Information Set + `_ explicitly excludes the attribute + order from conveying information. Code should be prepared to deal with + any ordering on input. In cases where deterministic XML output is required, + e.g. for cryptographic signing or test data sets, canonical serialisation + is available with the :func:`canonicalize` function. + + In cases where canonical output is not applicable but a specific attribute + order is still desirable on output, code should aim for creating the + attributes directly in the desired order, to avoid perceptual mismatches + for readers of the code. In cases where this is difficult to achieve, a + recipe like the following can be applied prior to serialisation to enforce + an order independently from the Element creation:: + + def reorder_attributes(root): + for el in root.iter(): + attrib = el.attrib + if len(attrib) > 1: + # adjust attribute order, e.g. by sorting + attribs = sorted(attrib.items()) + attrib.clear() + attrib.update(attribs) + .. _elementtree-elementtree-objects: @@ -904,12 +1107,6 @@ ElementTree Objects Same as :meth:`Element.findtext`, starting at the root of the tree. - .. method:: getiterator(tag=None) - - .. deprecated-removed:: 3.2 3.9 - Use method :meth:`ElementTree.iter` instead. - - .. method:: getroot() Returns the root element for this tree. @@ -1021,14 +1218,24 @@ TreeBuilder Objects ^^^^^^^^^^^^^^^^^^^ -.. class:: TreeBuilder(element_factory=None) +.. class:: TreeBuilder(element_factory=None, *, comment_factory=None, \ + pi_factory=None, insert_comments=False, insert_pis=False) Generic element structure builder. This builder converts a sequence of - start, data, and end method calls to a well-formed element structure. You - can use this class to build an element structure using a custom XML parser, - or a parser for some other XML-like format. *element_factory*, when given, - must be a callable accepting two positional arguments: a tag and - a dict of attributes. It is expected to return a new element instance. + start, data, end, comment and pi method calls to a well-formed element + structure. You can use this class to build an element structure using + a custom XML parser, or a parser for some other XML-like format. + + *element_factory*, when given, must be a callable accepting two positional + arguments: a tag and a dict of attributes. It is expected to return a new + element instance. + + The *comment_factory* and *pi_factory* functions, when given, should behave + like the :func:`Comment` and :func:`ProcessingInstruction` functions to + create comments and processing instructions. When not given, the default + factories will be used. When *insert_comments* and/or *insert_pis* is true, + comments/pis will be inserted into the tree if they appear within the root + element (but not outside of it). .. method:: close() @@ -1054,8 +1261,24 @@ TreeBuilder Objects containing element attributes. Returns the opened element. + .. method:: comment(text) + + Creates a comment with the given *text*. If ``insert_comments`` is true, + this will also add it to the tree. + + .. versionadded:: 3.8 + + + .. method:: pi(target, text) + + Creates a comment with the given *target* name and *text*. If + ``insert_pis`` is true, this will also add it to the tree. + + .. versionadded:: 3.8 + + In addition, a custom :class:`TreeBuilder` object can provide the - following method: + following methods: .. method:: doctype(name, pubid, system) @@ -1065,6 +1288,36 @@ TreeBuilder Objects .. versionadded:: 3.2 + .. method:: start_ns(prefix, uri) + + Is called whenever the parser encounters a new namespace declaration, + before the ``start()`` callback for the opening element that defines it. + *prefix* is ``''`` for the default namespace and the declared + namespace prefix name otherwise. *uri* is the namespace URI. + + .. versionadded:: 3.8 + + .. method:: end_ns(prefix) + + Is called after the ``end()`` callback of an element that declared + a namespace prefix mapping, with the name of the *prefix* that went + out of scope. + + .. versionadded:: 3.8 + + +.. class:: C14NWriterTarget(write, *, \ + with_comments=False, strip_text=False, rewrite_prefixes=False, \ + qname_aware_tags=None, qname_aware_attrs=None, \ + exclude_attrs=None, exclude_tags=None) + + A `C14N 2.0 `_ writer. Arguments are the + same as for the :func:`canonicalize` function. This class does not build a + tree but translates the callback events directly into a serialised form + using the *write* function. + + .. versionadded:: 3.8 + .. _elementtree-xmlparser-objects: @@ -1100,7 +1353,8 @@ XMLParser Objects :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method for each opening tag, its ``end(tag)`` method for each closing tag, and data - is processed by method ``data(data)``. :meth:`XMLParser.close` calls + is processed by method ``data(data)``. For further supported callback + methods, see the :class:`TreeBuilder` class. :meth:`XMLParser.close` calls *target*\'s method ``close()``. :class:`XMLParser` can be used not only for building a tree structure. This is an example of counting the maximum depth of an XML file:: @@ -1150,9 +1404,9 @@ XMLPullParser Objects callback target, :class:`XMLPullParser` collects an internal list of parsing events and lets the user read from it. *events* is a sequence of events to report back. The supported events are the strings ``"start"``, ``"end"``, - ``"start-ns"`` and ``"end-ns"`` (the "ns" events are used to get detailed - namespace information). If *events* is omitted, only ``"end"`` events are - reported. + ``"comment"``, ``"pi"``, ``"start-ns"`` and ``"end-ns"`` (the "ns" events + are used to get detailed namespace information). If *events* is omitted, + only ``"end"`` events are reported. .. method:: feed(data) @@ -1171,7 +1425,13 @@ XMLPullParser Objects data fed to the parser. The iterator yields ``(event, elem)`` pairs, where *event* is a string representing the type of event (e.g. ``"end"``) and *elem* is the - encountered :class:`Element` object. + encountered :class:`Element` object, or other context value as follows. + + * ``start``, ``end``: the current Element. + * ``comment``, ``pi``: the current comment / processing instruction + * ``start-ns``: a tuple ``(prefix, uri)`` naming the declared namespace + mapping. + * ``end-ns``: :const:`None` (this may change in a future version) Events provided in a previous call to :meth:`read_events` will not be yielded again. Events are consumed from the internal queue only when @@ -1191,6 +1451,10 @@ XMLPullParser Objects .. versionadded:: 3.4 + .. versionchanged:: 3.8 + The ``comment`` and ``pi`` events were added. + + Exceptions ^^^^^^^^^^ diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index fb86b6f5564d76..1981cab7cd438d 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -20,7 +20,7 @@ Python's interfaces for processing XML are grouped in the ``xml`` package. The XML modules are not secure against erroneous or maliciously constructed data. If you need to parse untrusted or unauthenticated data see the :ref:`xml-vulnerabilities` and - :ref:`defused-packages` sections. + :ref:`defusedxml-package` sections. It is important to note that modules in the :mod:`xml` package require that there be at least one SAX-compliant XML parser available. The Expat parser is @@ -113,9 +113,9 @@ decompression bomb The documentation for `defusedxml`_ on PyPI has further information about all known attack vectors with examples and references. -.. _defused-packages: +.. _defusedxml-package: -The :mod:`defusedxml` and :mod:`defusedexpat` Packages +The :mod:`defusedxml` Package ------------------------------------------------------ `defusedxml`_ is a pure Python package with modified subclasses of all stdlib @@ -124,16 +124,8 @@ package is recommended for any server code that parses untrusted XML data. The package also ships with example exploits and extended documentation on more XML exploits such as XPath injection. -`defusedexpat`_ provides a modified libexpat and a patched -:mod:`pyexpat` module that have countermeasures against entity expansion -DoS attacks. The :mod:`defusedexpat` module still allows a sane and configurable amount of entity -expansions. The modifications may be included in some future release of Python, -but will not be included in any bugfix releases of -Python because they break backward compatibility. - .. _defusedxml: https://pypi.org/project/defusedxml/ -.. _defusedexpat: https://pypi.org/project/defusedexpat/ .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb .. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index ae0877ca90db07..3746a58c9b9558 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -11,12 +11,12 @@ -------------- -The SAX API defines four kinds of handlers: content handlers, DTD handlers, -error handlers, and entity resolvers. Applications normally only need to -implement those interfaces whose events they are interested in; they can -implement the interfaces in a single object or in multiple objects. Handler -implementations should inherit from the base classes provided in the module -:mod:`xml.sax.handler`, so that all methods get default implementations. +The SAX API defines five kinds of handlers: content handlers, DTD handlers, +error handlers, entity resolvers and lexical handlers. Applications normally +only need to implement those interfaces whose events they are interested in; +they can implement the interfaces in a single object or in multiple objects. +Handler implementations should inherit from the base classes provided in the +module :mod:`xml.sax.handler`, so that all methods get default implementations. .. class:: ContentHandler @@ -47,6 +47,12 @@ implementations should inherit from the base classes provided in the module application. The methods of this object control whether errors are immediately converted to exceptions or are handled in some other way. + +.. class:: LexicalHandler + + Interface used by the parser to represent low freqency events which may not + be of interest to many applications. + In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants for the feature and property names. @@ -114,7 +120,7 @@ for the feature and property names. .. data:: property_lexical_handler | value: ``"http://xml.org/sax/properties/lexical-handler"`` - | data type: xml.sax.sax2lib.LexicalHandler (not supported in Python 2) + | data type: xml.sax.handler.LexicalHandler (not supported in Python 2) | description: An optional extension handler for lexical events like comments. | access: read/write @@ -413,3 +419,45 @@ the passed-in exception object. information will continue to be passed to the application. Raising an exception in this method will cause parsing to end. + +.. _lexical-handler-objects: + +LexicalHandler Objects +---------------------- +Optional SAX2 handler for lexical events. + +This handler is used to obtain lexical information about an XML +document. Lexical information includes information describing the +document encoding used and XML comments embedded in the document, as +well as section boundaries for the DTD and for any CDATA sections. +The lexical handlers are used in the same manner as content handlers. + +Set the LexicalHandler of an XMLReader by using the setProperty method +with the property identifier +``'http://xml.org/sax/properties/lexical-handler'``. + + +.. method:: LexicalHandler.comment(content) + + Reports a comment anywhere in the document (including the DTD and + outside the document element). + +.. method:: LexicalHandler.startDTD(name, public_id, system_id) + + Reports the start of the DTD declarations if the document has an + associated DTD. + +.. method:: LexicalHandler.endDTD() + + Reports the end of DTD declaration. + +.. method:: LexicalHandler.startCDATA() + + Reports the start of a CDATA marked section. + + The contents of the CDATA marked section will be reported through + the characters handler. + +.. method:: LexicalHandler.endCDATA() + + Reports the end of a CDATA marked section. diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 728315251e0824..fb40a2b3e964e4 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -198,7 +198,7 @@ Pack up a directory into an archive, and run it. The same can be done using the :func:`create_archive` function:: >>> import zipapp - >>> zipapp.create_archive('myapp.pyz', 'myapp') + >>> zipapp.create_archive('myapp', 'myapp.pyz') To make the application directly executable on POSIX, specify an interpreter to use. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 4e9edff270143d..7126d8bd703f62 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -52,6 +52,15 @@ The module defines the following items: :ref:`zipfile-objects` for constructor details. +.. class:: Path + :noindex: + + A pathlib-compatible wrapper for zip files. See section + :ref:`path-objects` for details. + + .. versionadded:: 3.8 + + .. class:: PyZipFile :noindex: @@ -456,6 +465,74 @@ The following data attributes are also available: truncated. +.. _path-objects: + +Path Objects +------------ + +.. class:: Path(root, at='') + + Construct a Path object from a ``root`` zipfile (which may be a + :class:`ZipFile` instance or ``file`` suitable for passing to + the :class:`ZipFile` constructor). + + ``at`` specifies the location of this Path within the zipfile, + e.g. 'dir/file.txt', 'dir/', or ''. Defaults to the empty string, + indicating the root. + +Path objects expose the following features of :mod:`pathlib.Path` +objects: + +Path objects are traversable using the ``/`` operator. + +.. attribute:: Path.name + + The final path component. + +.. method:: Path.open(mode='r', *, pwd, **) + + Invoke :meth:`ZipFile.open` on the current path. + Allows opening for read or write, text or binary + through supported modes: 'r', 'w', 'rb', 'wb'. + Positional and keyword arguments are passed through to + :class:`io.TextIOWrapper` when opened as text and + ignored otherwise. + ``pwd`` is the ``pwd`` parameter to + :meth:`ZipFile.open`. + + .. versionchanged:: 3.9 + Added support for text and binary modes for open. Default + mode is now text. + +.. method:: Path.iterdir() + + Enumerate the children of the current directory. + +.. method:: Path.is_dir() + + Return ``True`` if the current context references a directory. + +.. method:: Path.is_file() + + Return ``True`` if the current context references a file. + +.. method:: Path.exists() + + Return ``True`` if the current context references a file or + directory in the zip file. + +.. method:: Path.read_text(*, **) + + Read the current file as unicode text. Positional and + keyword arguments are passed through to + :class:`io.TextIOWrapper` (except ``buffer``, which is + implied by the context). + +.. method:: Path.read_bytes() + + Read the current file as bytes. + + .. _pyzipfile-objects: PyZipFile Objects @@ -749,5 +826,45 @@ Command-line options Test whether the zipfile is valid or not. +Decompression pitfalls +---------------------- + +The extraction in zipfile module might fail due to some pitfalls listed below. + +From file itself +~~~~~~~~~~~~~~~~ + +Decompression may fail due to incorrect password / CRC checksum / ZIP format or +unsupported compression method / decryption. + +File System limitations +~~~~~~~~~~~~~~~~~~~~~~~ + +Exceeding limitations on different file systems can cause decompression failed. +Such as allowable characters in the directory entries, length of the file name, +length of the pathname, size of a single file, and number of files, etc. + +Resources limitations +~~~~~~~~~~~~~~~~~~~~~ + +The lack of memory or disk volume would lead to decompression +failed. For example, decompression bombs (aka `ZIP bomb`_) +apply to zipfile library that can cause disk volume exhaustion. + +Interruption +~~~~~~~~~~~~ + +Interruption during the decompression, such as pressing control-C or killing the +decompression process may result in incomplete decompression of the archive. + +Default behaviors of extraction +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Not knowing the default extraction behaviors +can cause unexpected decompression results. +For example, when extracting the same archive twice, +it overwrites files without asking. + +.. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb .. _PKZIP Application Note: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index aa1831dbc60de9..8ac3fb16bdb90d 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -2,10 +2,12 @@ ===================================================== .. module:: zipimport - :synopsis: support for importing Python modules from ZIP archives. + :synopsis: Support for importing Python modules from ZIP archives. .. moduleauthor:: Just van Rossum +**Source code:** :source:`Lib/zipimport.py` + -------------- This module adds the ability to import Python modules (:file:`\*.py`, diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index aa61278e099ac6..ec60ea24db6627 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -9,9 +9,9 @@ For applications that require data compression, the functions in this module allow compression and decompression, using the zlib library. The zlib library -has its own home page at http://www.zlib.net. There are known +has its own home page at https://www.zlib.net. There are known incompatibilities between the Python module and versions of the zlib library -earlier than 1.1.3; 1.1.3 has a security vulnerability, so we recommend using +earlier than 1.1.3; 1.1.3 has a `security vulnerability `_, so we recommend using 1.1.4 or later. zlib's functions have many options and often need to be used in a particular @@ -47,7 +47,7 @@ The available exception and functions in this module are: platforms, use ``adler32(data) & 0xffffffff``. -.. function:: compress(data, level=-1) +.. function:: compress(data, /, level=-1) Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; @@ -132,7 +132,7 @@ The available exception and functions in this module are: platforms, use ``crc32(data) & 0xffffffff``. -.. function:: decompress(data, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) +.. function:: decompress(data, /, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) Decompresses the bytes in *data*, returning a bytes object containing the uncompressed data. The *wbits* parameter depends on @@ -337,4 +337,3 @@ the following constants: http://www.zlib.net/manual.html The zlib manual explains the semantics and usage of the library's many functions. - diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst new file mode 100644 index 00000000000000..3a4c12a73acd72 --- /dev/null +++ b/Doc/library/zoneinfo.rst @@ -0,0 +1,413 @@ +:mod:`zoneinfo` --- IANA time zone support +========================================== + +.. module:: zoneinfo + :synopsis: IANA time zone support + +.. versionadded:: 3.9 + +.. moduleauthor:: Paul Ganssle +.. sectionauthor:: Paul Ganssle + +-------------- + +The :mod:`zoneinfo` module provides a concrete time zone implementation to +support the IANA time zone database as originally specified in :pep:`615`. By +default, :mod:`zoneinfo` uses the system's time zone data if available; if no +system time zone data is available, the library will fall back to using the +first-party `tzdata`_ package available on PyPI. + +.. seealso:: + + Module: :mod:`datetime` + Provides the :class:`~datetime.time` and :class:`~datetime.datetime` + types with which the :class:`ZoneInfo` class is designed to be used. + + Package `tzdata`_ + First-party package maintained by the CPython core developers to supply + time zone data via PyPI. + + +Using ``ZoneInfo`` +------------------ + +:class:`ZoneInfo` is a concrete implementation of the :class:`datetime.tzinfo` +abstract base class, and is intended to be attached to ``tzinfo``, either via +the constructor, the :meth:`datetime.replace ` +method or :meth:`datetime.astimezone `:: + + >>> from zoneinfo import ZoneInfo + >>> from datetime import datetime, timedelta + + >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(dt) + 2020-10-31 12:00:00-07:00 + + >>> dt.tzname() + 'PDT' + +Datetimes constructed in this way are compatible with datetime arithmetic and +handle daylight saving time transitions with no further intervention:: + + >>> dt_add = dt + timedelta(days=1) + + >>> print(dt_add) + 2020-11-01 12:00:00-08:00 + + >>> dt_add.tzname() + 'PST' + +These time zones also support the :attr:`~datetime.datetime.fold` attribute +introduced in :pep:`495`. During offset transitions which induce ambiguous +times (such as a daylight saving time to standard time transition), the offset +from *before* the transition is used when ``fold=0``, and the offset *after* +the transition is used when ``fold=1``, for example:: + + >>> dt = datetime(2020, 11, 1, 1, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(dt) + 2020-11-01 01:00:00-07:00 + + >>> print(dt.replace(fold=1)) + 2020-11-01 01:00:00-08:00 + +When converting from another time zone, the fold will be set to the correct +value:: + + >>> from datetime import timezone + >>> LOS_ANGELES = ZoneInfo("America/Los_Angeles") + >>> dt_utc = datetime(2020, 11, 1, 8, tzinfo=timezone.utc) + + >>> # Before the PDT -> PST transition + >>> print(dt_utc.astimezone(LOS_ANGELES)) + 2020-11-01 01:00:00-07:00 + + >>> # After the PDT -> PST transition + >>> print((dt_utc + timedelta(hours=1)).astimezone(LOS_ANGELES)) + 2020-11-01 01:00:00-08:00 + +Data sources +------------ + +The ``zoneinfo`` module does not directly provide time zone data, and instead +pulls time zone information from the system time zone database or the +first-party PyPI package `tzdata`_, if available. Some systems, including +notably Windows systems, do not have an IANA database available, and so for +projects targeting cross-platform compatibility that require time zone data, it +is recommended to declare a dependency on tzdata. If neither system data nor +tzdata are available, all calls to :class:`ZoneInfo` will raise +:exc:`ZoneInfoNotFoundError`. + +.. _zoneinfo_data_configuration: + +Configuring the data sources +**************************** + +When ``ZoneInfo(key)`` is called, the constructor first searches the +directories specified in :data:`TZPATH` for a file matching ``key``, and on +failure looks for a match in the tzdata package. This behavior can be +configured in three ways: + +1. The default :data:`TZPATH` when not otherwise specified can be configured at + :ref:`compile time `. +2. :data:`TZPATH` can be configured using :ref:`an environment variable + `. +3. At :ref:`runtime `, the search path can be + manipulated using the :func:`reset_tzpath` function. + +.. _zoneinfo_data_compile_time_config: + +Compile-time configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default :data:`TZPATH` includes several common deployment locations for the +time zone database (except on Windows, where there are no "well-known" +locations for time zone data). On POSIX systems, downstream distributors and +those building Python from source who know where their system +time zone data is deployed may change the default time zone path by specifying +the compile-time option ``TZPATH`` (or, more likely, the ``configure`` flag +``--with-tzpath``), which should be a string delimited by :data:`os.pathsep`. + +On all platforms, the configured value is available as the ``TZPATH`` key in +:func:`sysconfig.get_config_var`. + +.. _zoneinfo_data_environment_var: + +Environment configuration +^^^^^^^^^^^^^^^^^^^^^^^^^ + +When initializing :data:`TZPATH` (either at import time or whenever +:func:`reset_tzpath` is called with no arguments), the ``zoneinfo`` module will +use the environment variable ``PYTHONTZPATH``, if it exists, to set the search +path. + +.. envvar:: PYTHONTZPATH + + This is an :data:`os.pathsep`-separated string containing the time zone + search path to use. It must consist of only absolute rather than relative + paths. Relative components specified in ``PYTHONTZPATH`` will not be used, + but otherwise the behavior when a relative path is specified is + implementation-defined; CPython will raise :exc:`InvalidTZPathWarning`, but + other implementations are free to silently ignore the erroneous component + or raise an exception. + +To set the system to ignore the system data and use the tzdata package +instead, set ``PYTHONTZPATH=""``. + +.. _zoneinfo_data_runtime_config: + +Runtime configuration +^^^^^^^^^^^^^^^^^^^^^ + +The TZ search path can also be configured at runtime using the +:func:`reset_tzpath` function. This is generally not an advisable operation, +though it is reasonable to use it in test functions that require the use of a +specific time zone path (or require disabling access to the system time zones). + + +The ``ZoneInfo`` class +---------------------- + +.. class:: ZoneInfo(key) + + A concrete :class:`datetime.tzinfo` subclass that represents an IANA time + zone specified by the string ``key``. Calls to the primary constructor will + always return objects that compare identically; put another way, barring + cache invalidation via :meth:`ZoneInfo.clear_cache`, for all values of + ``key``, the following assertion will always be true: + + .. code-block:: python + + a = ZoneInfo(key) + b = ZoneInfo(key) + assert a is b + + ``key`` must be in the form of a relative, normalized POSIX path, with no + up-level references. The constructor will raise :exc:`ValueError` if a + non-conforming key is passed. + + If no file matching ``key`` is found, the constructor will raise + :exc:`ZoneInfoNotFoundError`. + + +The ``ZoneInfo`` class has two alternate constructors: + +.. classmethod:: ZoneInfo.from_file(fobj, /, key=None) + + Constructs a ``ZoneInfo`` object from a file-like object returning bytes + (e.g. a file opened in binary mode or an :class:`io.BytesIO` object). + Unlike the primary constructor, this always constructs a new object. + + The ``key`` parameter sets the name of the zone for the purposes of + :py:meth:`~object.__str__` and :py:meth:`~object.__repr__`. + + Objects created via this constructor cannot be pickled (see `pickling`_). + +.. classmethod:: ZoneInfo.no_cache(key) + + An alternate constructor that bypasses the constructor's cache. It is + identical to the primary constructor, but returns a new object on each + call. This is most likely to be useful for testing or demonstration + purposes, but it can also be used to create a system with a different cache + invalidation strategy. + + Objects created via this constructor will also bypass the cache of a + deserializing process when unpickled. + + .. TODO: Add "See `cache_behavior`_" reference when that section is ready. + + .. caution:: + + Using this constructor may change the semantics of your datetimes in + surprising ways, only use it if you know that you need to. + +The following class methods are also available: + +.. classmethod:: ZoneInfo.clear_cache(*, only_keys=None) + + A method for invalidating the cache on the ``ZoneInfo`` class. If no + arguments are passed, all caches are invalidated and the next call to + the primary constructor for each key will return a new instance. + + If an iterable of key names is passed to the ``only_keys`` parameter, only + the specified keys will be removed from the cache. Keys passed to + ``only_keys`` but not found in the cache are ignored. + + .. TODO: Add "See `cache_behavior`_" reference when that section is ready. + + .. warning:: + + Invoking this function may change the semantics of datetimes using + ``ZoneInfo`` in surprising ways; this modifies process-wide global state + and thus may have wide-ranging effects. Only use it if you know that you + need to. + +The class has one attribute: + +.. attribute:: ZoneInfo.key + + This is a read-only :term:`attribute` that returns the value of ``key`` + passed to the constructor, which should be a lookup key in the IANA time + zone database (e.g. ``America/New_York``, ``Europe/Paris`` or + ``Asia/Tokyo``). + + For zones constructed from file without specifying a ``key`` parameter, + this will be set to ``None``. + + .. note:: + + Although it is a somewhat common practice to expose these to end users, + these values are designed to be primary keys for representing the + relevant zones and not necessarily user-facing elements. Projects like + CLDR (the Unicode Common Locale Data Repository) can be used to get + more user-friendly strings from these keys. + +String representations +********************** + +The string representation returned when calling :py:class:`str` on a +:class:`ZoneInfo` object defaults to using the :attr:`ZoneInfo.key` attribute (see +the note on usage in the attribute documentation):: + + >>> zone = ZoneInfo("Pacific/Kwajalein") + >>> str(zone) + 'Pacific/Kwajalein' + + >>> dt = datetime(2020, 4, 1, 3, 15, tzinfo=zone) + >>> f"{dt.isoformat()} [{dt.tzinfo}]" + '2020-04-01T03:15:00+12:00 [Pacific/Kwajalein]' + +For objects constructed from a file without specifying a ``key`` parameter, +``str`` falls back to calling :func:`repr`. ``ZoneInfo``'s ``repr`` is +implementation-defined and not necessarily stable between versions, but it is +guaranteed not to be a valid ``ZoneInfo`` key. + +.. _pickling: + +Pickle serialization +******************** + +Rather than serializing all transition data, ``ZoneInfo`` objects are +serialized by key, and ``ZoneInfo`` objects constructed from files (even those +with a value for ``key`` specified) cannot be pickled. + +The behavior of a ``ZoneInfo`` file depends on how it was constructed: + +1. ``ZoneInfo(key)``: When constructed with the primary constructor, a + ``ZoneInfo`` object is serialized by key, and when deserialized, the + deserializing process uses the primary and thus it is expected that these + are expected to be the same object as other references to the same time + zone. For example, if ``europe_berlin_pkl`` is a string containing a pickle + constructed from ``ZoneInfo("Europe/Berlin")``, one would expect the + following behavior: + + .. code-block:: pycon + + >>> a = ZoneInfo("Europe/Berlin") + >>> b = pickle.loads(europe_berlin_pkl) + >>> a is b + True + +2. ``ZoneInfo.no_cache(key)``: When constructed from the cache-bypassing + constructor, the ``ZoneInfo`` object is also serialized by key, but when + deserialized, the deserializing process uses the cache bypassing + constructor. If ``europe_berlin_pkl_nc`` is a string containing a pickle + constructed from ``ZoneInfo.no_cache("Europe/Berlin")``, one would expect + the following behavior: + + .. code-block:: pycon + + >>> a = ZoneInfo("Europe/Berlin") + >>> b = pickle.loads(europe_berlin_pkl_nc) + >>> a is b + False + +3. ``ZoneInfo.from_file(fobj, /, key=None)``: When constructed from a file, the + ``ZoneInfo`` object raises an exception on pickling. If an end user wants to + pickle a ``ZoneInfo`` constructed from a file, it is recommended that they + use a wrapper type or a custom serialization function: either serializing by + key or storing the contents of the file object and serializing that. + +This method of serialization requires that the time zone data for the required +key be available on both the serializing and deserializing side, similar to the +way that references to classes and functions are expected to exist in both the +serializing and deserializing environments. It also means that no guarantees +are made about the consistency of results when unpickling a ``ZoneInfo`` +pickled in an environment with a different version of the time zone data. + +Functions +--------- + +.. function:: available_timezones() + + Get a set containing all the valid keys for IANA time zones available + anywhere on the time zone path. This is recalculated on every call to the + function. + + This function only includes canonical zone names and does not include + "special" zones such as those under the ``posix/`` and ``right/`` + directories, or the ``posixrules`` zone. + + .. caution:: + + This function may open a large number of files, as the best way to + determine if a file on the time zone path is a valid time zone is to + read the "magic string" at the beginning. + + .. note:: + + These values are not designed to be exposed to end-users; for user + facing elements, applications should use something like CLDR (the + Unicode Common Locale Data Repository) to get more user-friendly + strings. See also the cautionary note on :attr:`ZoneInfo.key`. + +.. function:: reset_tzpath(to=None) + + Sets or resets the time zone search path (:data:`TZPATH`) for the module. + When called with no arguments, :data:`TZPATH` is set to the default value. + + Calling ``reset_tzpath`` will not invalidate the :class:`ZoneInfo` cache, + and so calls to the primary ``ZoneInfo`` constructor will only use the new + ``TZPATH`` in the case of a cache miss. + + The ``to`` parameter must be a :term:`sequence` of strings or + :class:`os.PathLike` and not a string, all of which must be absolute paths. + :exc:`ValueError` will be raised if something other than an absolute path + is passed. + +Globals +------- + +.. data:: TZPATH + + A read-only sequence representing the time zone search path -- when + constructing a ``ZoneInfo`` from a key, the key is joined to each entry in + the ``TZPATH``, and the first file found is used. + + ``TZPATH`` may contain only absolute paths, never relative paths, + regardless of how it is configured. + + The object that ``zoneinfo.TZPATH`` points to may change in response to a + call to :func:`reset_tzpath`, so it is recommended to use + ``zoneinfo.TZPATH`` rather than importing ``TZPATH`` from ``zoneinfo`` or + assigning a long-lived variable to ``zoneinfo.TZPATH``. + + For more information on configuring the time zone search path, see + :ref:`zoneinfo_data_configuration`. + +Exceptions and warnings +----------------------- + +.. exception:: ZoneInfoNotFoundError + + Raised when construction of a :class:`ZoneInfo` object fails because the + specified key could not be found on the system. This is a subclass of + :exc:`KeyError`. + +.. exception:: InvalidTZPathWarning + + Raised when :envvar:`PYTHONTZPATH` contains an invalid component that will + be filtered out, such as a relative path. + +.. Links and references: + +.. _tzdata: https://pypi.org/project/tzdata/ diff --git a/Doc/license.rst b/Doc/license.rst index a315b6f8134d48..4030825bbd28ee 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -1,4 +1,4 @@ -.. highlightlang:: none +.. highlight:: none .. _history-and-license: @@ -22,7 +22,7 @@ Virginia where he released several versions of the software. In May 2000, Guido and the Python core development team moved to BeOpen.com to form the BeOpen PythonLabs team. In October of the same year, the PythonLabs team moved to Digital Creations (now Zope Corporation; see -http://www.zope.com/). In 2001, the Python Software Foundation (PSF, see +https://www.zope.org/). In 2001, the Python Software Foundation (PSF, see https://www.python.org/psf/) was formed, a non-profit organization created specifically to own Python-related Intellectual Property. Zope Corporation is a sponsoring member of the PSF. @@ -72,6 +72,19 @@ make these releases possible. Terms and conditions for accessing or otherwise using Python ============================================================ +Python software and documentation are licensed under the +:ref:`PSF License Agreement `. + +Starting with Python 3.8.6, examples, recipes, and other code in +the documentation are dual licensed under the PSF License Agreement +and the :ref:`Zero-Clause BSD license `. + +Some software incorporated into Python is under different licenses. +The licenses are listed with code falling under that license. +See :ref:`OtherLicenses` for an incomplete list of these licenses. + + +.. _PSF-license: PSF LICENSE AGREEMENT FOR PYTHON |release| ------------------------------------------ @@ -87,7 +100,7 @@ PSF LICENSE AGREEMENT FOR PYTHON |release| analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python |release| alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2019 Python Software Foundation; All Rights + copyright, i.e., "Copyright © 2001-2020 Python Software Foundation; All Rights Reserved" are retained in Python |release| alone or in any derivative version prepared by Licensee. @@ -258,6 +271,27 @@ CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 SOFTWARE. +.. _BSD0: + +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON |release| DOCUMENTATION +---------------------------------------------------------------------- + +.. parsed-literal:: + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THIS SOFTWARE. + + +.. _OtherLicenses: + Licenses and Acknowledgements for Incorporated Software ======================================================= @@ -561,7 +595,7 @@ SipHash24 --------- The file :file:`Python/pyhash.c` contains Marek Majkowski' implementation of -Dan Bernstein's SipHash24 algorithm. The contains the following note:: +Dan Bernstein's SipHash24 algorithm. It contains the following note:: Copyright (c) 2013 Marek Majkowski @@ -889,7 +923,7 @@ libmpdec The :mod:`_decimal` module is built using an included copy of the libmpdec library unless the build is configured ``--with-system-libmpdec``:: - Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + Copyright (c) 2008-2020 Stefan Krah. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -913,3 +947,40 @@ library unless the build is configured ``--with-system-libmpdec``:: LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +W3C C14N test suite +------------------- + +The C14N 2.0 test suite in the :mod:`test` package +(``Lib/test/xmltestdata/c14n-20/``) was retrieved from the W3C website at +https://www.w3.org/TR/xml-c14n2-testcases/ and is distributed under the +3-clause BSD license: + + Copyright (c) 2013 W3C(R) (MIT, ERCIM, Keio, Beihang), + All Rights Reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of works must retain the original copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the original copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the W3C nor the names of its contributors may be + used to endorse or promote products derived from this work without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Doc/make.bat b/Doc/make.bat index e6604956ea916b..7fde0636427713 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -13,7 +13,7 @@ if not defined SPHINXBUILD ( %PYTHON% -c "import sphinx" > nul 2> nul if errorlevel 1 ( echo Installing sphinx with %PYTHON% - %PYTHON% -m pip install sphinx + %PYTHON% -m pip install sphinx==2.2.0 if errorlevel 1 exit /B ) set SPHINXBUILD=%PYTHON% -c "import sphinx.cmd.build, sys; sys.exit(sphinx.cmd.build.main())" @@ -54,9 +54,9 @@ if not exist "%HTMLHELP%" ( ) :skiphhcsearch -if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v +if not defined DISTVERSION for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v -if "%BUILDDIR%" EQU "" set BUILDDIR=build +if not defined BUILDDIR set BUILDDIR=build rem Targets that don't require sphinx-build if "%1" EQU "" goto help @@ -117,13 +117,13 @@ if not exist "%BUILDDIR%" mkdir "%BUILDDIR%" rem PY_MISC_NEWS_DIR is also used by our Sphinx extension in tools/extensions/pyspecific.py if not defined PY_MISC_NEWS_DIR set PY_MISC_NEWS_DIR=%BUILDDIR%\%1 +if not exist "%PY_MISC_NEWS_DIR%" mkdir "%PY_MISC_NEWS_DIR%" if exist ..\Misc\NEWS ( echo.Copying Misc\NEWS to %PY_MISC_NEWS_DIR%\NEWS copy ..\Misc\NEWS "%PY_MISC_NEWS_DIR%\NEWS" > nul ) else if exist ..\Misc\NEWS.D ( if defined BLURB ( echo.Merging Misc/NEWS with %BLURB% - if not exist build mkdir build %BLURB% merge -f "%PY_MISC_NEWS_DIR%\NEWS" ) else ( echo.No Misc/NEWS file and Blurb is not available. @@ -131,7 +131,7 @@ if exist ..\Misc\NEWS ( ) ) -if NOT "%PAPER%" == "" ( +if defined PAPER ( set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% ) if "%1" EQU "htmlhelp" ( diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 42fa8647623935..158d6a8f164e23 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -44,7 +44,8 @@ executed:: Summarizing: -.. productionlist:: + +.. productionlist:: python-grammar compound_stmt: `if_stmt` : | `while_stmt` : | `for_stmt` @@ -89,9 +90,9 @@ The :keyword:`!if` statement The :keyword:`if` statement is used for conditional execution: -.. productionlist:: - if_stmt: "if" `expression` ":" `suite` - : ("elif" `expression` ":" `suite`)* +.. productionlist:: python-grammar + if_stmt: "if" `assignment_expression` ":" `suite` + : ("elif" `assignment_expression` ":" `suite`)* : ["else" ":" `suite`] It selects exactly one of the suites by evaluating the expressions one by one @@ -115,8 +116,8 @@ The :keyword:`!while` statement The :keyword:`while` statement is used for repeated execution as long as an expression is true: -.. productionlist:: - while_stmt: "while" `expression` ":" `suite` +.. productionlist:: python-grammar + while_stmt: "while" `assignment_expression` ":" `suite` : ["else" ":" `suite`] This repeatedly tests the expression and, if it is true, executes the first @@ -151,7 +152,7 @@ The :keyword:`!for` statement The :keyword:`for` statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: -.. productionlist:: +.. productionlist:: python-grammar for_stmt: "for" `target_list` "in" `expression_list` ":" `suite` : ["else" ":" `suite`] @@ -234,7 +235,7 @@ The :keyword:`!try` statement The :keyword:`try` statement specifies exception handlers and/or cleanup code for a group of statements: -.. productionlist:: +.. productionlist:: python-grammar try_stmt: `try1_stmt` | `try2_stmt` try1_stmt: "try" ":" `suite` : ("except" [`expression` ["as" `identifier`]] ":" `suite`)+ @@ -390,7 +391,7 @@ methods defined by a context manager (see section :ref:`context-managers`). This allows common :keyword:`try`...\ :keyword:`except`...\ :keyword:`finally` usage patterns to be encapsulated for convenient reuse. -.. productionlist:: +.. productionlist:: python-grammar with_stmt: "with" `with_item` ("," `with_item`)* ":" `suite` with_item: `expression` ["as" `target`] @@ -399,6 +400,8 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo #. The context expression (the expression given in the :token:`with_item`) is evaluated to obtain a context manager. +#. The context manager's :meth:`__enter__` is loaded for later use. + #. The context manager's :meth:`__exit__` is loaded for later use. #. The context manager's :meth:`__enter__` method is invoked. @@ -430,17 +433,41 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo value from :meth:`__exit__` is ignored, and execution proceeds at the normal location for the kind of exit that was taken. +The following code:: + + with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to:: + + manager = (EXPRESSION) + enter = type(manager).__enter__ + exit = type(manager).__exit__ + value = enter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not exit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + exit(manager, None, None, None) + With more than one item, the context managers are processed as if multiple :keyword:`with` statements were nested:: with A() as a, B() as b: - suite + SUITE -is equivalent to :: +is semantically equivalent to:: with A() as a: with B() as b: - suite + SUITE .. versionchanged:: 3.1 Support for multiple context expressions. @@ -477,14 +504,16 @@ Function definitions A function definition defines a user-defined function object (see section :ref:`types`): -.. productionlist:: +.. productionlist:: python-grammar funcdef: [`decorators`] "def" `funcname` "(" [`parameter_list`] ")" : ["->" `expression`] ":" `suite` decorators: `decorator`+ - decorator: "@" `dotted_name` ["(" [`argument_list` [","]] ")"] NEWLINE + decorator: "@" `assignment_expression` NEWLINE dotted_name: `identifier` ("." `identifier`)* - parameter_list: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] - : | `parameter_list_starargs` + parameter_list: `defparameter` ("," `defparameter`)* "," "/" ["," [`parameter_list_no_posonly`]] + : | `parameter_list_no_posonly` + parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] + : | `parameter_list_starargs` parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] @@ -522,6 +551,11 @@ is roughly equivalent to :: except that the original function is not temporarily bound to the name ``func``. +.. versionchanged:: 3.9 + Functions may be decorated with any valid :token:`assignment_expression`. + Previously, the grammar was much more restrictive; see :pep:`614` for + details. + .. index:: triple: default; parameter; value single: argument; function definition @@ -538,9 +572,9 @@ value --- this is a syntactic restriction that is not expressed by the grammar. **Default parameter values are evaluated from left to right when the function definition is executed.** This means that the expression is evaluated once, when the function is defined, and that the same "pre-computed" value is used for each -call. This is especially important to understand when a default parameter is a +call. This is especially important to understand when a default parameter value is a mutable object, such as a list or a dictionary: if the function modifies the -object (e.g. by appending an item to a list), the default value is in effect +object (e.g. by appending an item to a list), the default parameter value is in effect modified. This is generally not what was intended. A way around this is to use ``None`` as the default, and explicitly test for it in the body of the function, e.g.:: @@ -637,7 +671,7 @@ Class definitions A class definition defines a class object (see section :ref:`types`): -.. productionlist:: +.. productionlist:: python-grammar classdef: [`decorators`] "class" `classname` [`inheritance`] ":" `suite` inheritance: "(" [`argument_list`] ")" classname: `identifier` @@ -689,6 +723,11 @@ is roughly equivalent to :: The evaluation rules for the decorator expressions are the same as for function decorators. The result is then bound to the class name. +.. versionchanged:: 3.9 + Classes may be decorated with any valid :token:`assignment_expression`. + Previously, the grammar was much more restrictive; see :pep:`614` for + details. + **Programmer's note:** Variables defined in the class definition are class attributes; they are shared by instances. Instance attributes can be set in a method with ``self.name = value``. Both class and instance attributes are @@ -724,7 +763,7 @@ Coroutines Coroutine function definition ----------------------------- -.. productionlist:: +.. productionlist:: python-grammar async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")" : ["->" `expression`] ":" `suite` @@ -757,7 +796,7 @@ An example of a coroutine function:: The :keyword:`!async for` statement ----------------------------------- -.. productionlist:: +.. productionlist:: python-grammar async_for_stmt: "async" `for_stmt` An :term:`asynchronous iterable` is able to call asynchronous code in its @@ -770,24 +809,25 @@ iterators. The following code:: async for TARGET in ITER: - BLOCK + SUITE else: - BLOCK2 + SUITE2 Is semantically equivalent to:: iter = (ITER) iter = type(iter).__aiter__(iter) running = True + while running: try: TARGET = await type(iter).__anext__(iter) except StopAsyncIteration: running = False else: - BLOCK + SUITE else: - BLOCK2 + SUITE2 See also :meth:`__aiter__` and :meth:`__anext__` for details. @@ -801,7 +841,7 @@ body of a coroutine function. The :keyword:`!async with` statement ------------------------------------ -.. productionlist:: +.. productionlist:: python-grammar async_with_stmt: "async" `with_stmt` An :term:`asynchronous context manager` is a :term:`context manager` that is @@ -809,23 +849,27 @@ able to suspend execution in its *enter* and *exit* methods. The following code:: - async with EXPR as VAR: - BLOCK + async with EXPRESSION as TARGET: + SUITE -Is semantically equivalent to:: +is semantically equivalent to:: - mgr = (EXPR) - aexit = type(mgr).__aexit__ - aenter = type(mgr).__aenter__(mgr) + manager = (EXPRESSION) + aenter = type(manager).__aenter__ + aexit = type(manager).__aexit__ + value = await aenter(manager) + hit_except = False - VAR = await aenter try: - BLOCK + TARGET = value + SUITE except: - if not await aexit(mgr, *sys.exc_info()): + hit_except = True + if not await aexit(manager, *sys.exc_info()): raise - else: - await aexit(mgr, None, None, None) + finally: + if not hit_except: + await aexit(manager, None, None, None) See also :meth:`__aenter__` and :meth:`__aexit__` for details. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 1683d25db9247a..a817408c3b1ef5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -17,7 +17,7 @@ Objects, values and types :dfn:`Objects` are Python's abstraction for data. All data in a Python program is represented by objects or by relations between objects. (In a sense, and in -conformance to Von Neumann's model of a "stored program computer," code is also +conformance to Von Neumann's model of a "stored program computer", code is also represented by objects.) .. index:: @@ -156,13 +156,18 @@ NotImplemented object is accessed through the built-in name ``NotImplemented``. Numeric methods and rich comparison methods should return this value if they do not implement the operation for the operands provided. (The interpreter will then try the - reflected operation, or some other fallback, depending on the operator.) Its - truth value is true. + reflected operation, or some other fallback, depending on the operator.) It + should not be evaluated in a boolean context. See :ref:`implementing-the-arithmetic-operations` for more details. + .. versionchanged:: 3.9 + Evaluating ``NotImplemented`` in a boolean context is deprecated. While + it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. + It will raise a :exc:`TypeError` in a future version of Python. + Ellipsis .. index:: @@ -420,6 +425,11 @@ Mappings equal (e.g., ``1`` and ``1.0``) then they can be used interchangeably to index the same dictionary entry. + Dictionaries preserve insertion order, meaning that keys will be produced + in the same order they were added sequentially over the dictionary. + Replacing an existing key does not change the order, however removing a key + and re-inserting it will add it to the end instead of keeping its old place. + Dictionaries are mutable; they can be created by the ``{...}`` notation (see section :ref:`dict`). @@ -431,6 +441,11 @@ Mappings additional examples of mapping types, as does the :mod:`collections` module. + .. versionchanged:: 3.7 + Dictionaries did not preserve insertion order in versions of Python before 3.6. + In CPython 3.6, insertion order was preserved, but it was considered + an implementation detail at that time rather than a language guarantee. + Callable types .. index:: object: callable @@ -890,6 +905,8 @@ Internal types .. index:: single: co_argcount (code object attribute) + single: co_posonlyargcount (code object attribute) + single: co_kwonlyargcount (code object attribute) single: co_code (code object attribute) single: co_consts (code object attribute) single: co_filename (code object attribute) @@ -905,21 +922,26 @@ Internal types single: co_freevars (code object attribute) Special read-only attributes: :attr:`co_name` gives the function name; - :attr:`co_argcount` is the number of positional arguments (including arguments - with default values); :attr:`co_nlocals` is the number of local variables used - by the function (including arguments); :attr:`co_varnames` is a tuple containing + :attr:`co_argcount` is the total number of positional arguments + (including positional-only arguments and arguments with default values); + :attr:`co_posonlyargcount` is the number of positional-only arguments + (including arguments with default values); :attr:`co_kwonlyargcount` is + the number of keyword-only arguments (including arguments with default + values); :attr:`co_nlocals` is the number of local variables used by the + function (including arguments); :attr:`co_varnames` is a tuple containing the names of the local variables (starting with the argument names); - :attr:`co_cellvars` is a tuple containing the names of local variables that are - referenced by nested functions; :attr:`co_freevars` is a tuple containing the - names of free variables; :attr:`co_code` is a string representing the sequence - of bytecode instructions; :attr:`co_consts` is a tuple containing the literals - used by the bytecode; :attr:`co_names` is a tuple containing the names used by - the bytecode; :attr:`co_filename` is the filename from which the code was - compiled; :attr:`co_firstlineno` is the first line number of the function; - :attr:`co_lnotab` is a string encoding the mapping from bytecode offsets to - line numbers (for details see the source code of the interpreter); - :attr:`co_stacksize` is the required stack size (including local variables); - :attr:`co_flags` is an integer encoding a number of flags for the interpreter. + :attr:`co_cellvars` is a tuple containing the names of local variables + that are referenced by nested functions; :attr:`co_freevars` is a tuple + containing the names of free variables; :attr:`co_code` is a string + representing the sequence of bytecode instructions; :attr:`co_consts` is + a tuple containing the literals used by the bytecode; :attr:`co_names` is + a tuple containing the names used by the bytecode; :attr:`co_filename` is + the filename from which the code was compiled; :attr:`co_firstlineno` is + the first line number of the function; :attr:`co_lnotab` is a string + encoding the mapping from bytecode offsets to line numbers (for details + see the source code of the interpreter); :attr:`co_stacksize` is the + required stack size; :attr:`co_flags` is an integer encoding a number + of flags for the interpreter. .. index:: object: generator @@ -1159,10 +1181,10 @@ Basic customization with appropriate arguments and then modifying the newly-created instance as necessary before returning it. - If :meth:`__new__` returns an instance of *cls*, then the new instance's - :meth:`__init__` method will be invoked like ``__init__(self[, ...])``, where - *self* is the new instance and the remaining arguments are the same as were - passed to :meth:`__new__`. + If :meth:`__new__` is invoked during object construction and it returns an + instance or subclass of *cls*, then the new instance’s :meth:`__init__` method + will be invoked like ``__init__(self[, ...])``, where *self* is the new instance + and the remaining arguments are the same as were passed to the object constructor. If :meth:`__new__` does not return an instance of *cls*, then the new instance's :meth:`__init__` method will not be invoked. @@ -1311,9 +1333,9 @@ Basic customization Called by the :func:`format` built-in function, and by extension, evaluation of :ref:`formatted string literals ` and the :meth:`str.format` method, to produce a "formatted" - string representation of an object. The ``format_spec`` argument is + string representation of an object. The *format_spec* argument is a string that contains a description of the formatting options desired. - The interpretation of the ``format_spec`` argument is up to the type + The interpretation of the *format_spec* argument is up to the type implementing :meth:`__format__`, however most classes will either delegate formatting to one of the built-in types, or use a similar formatting option syntax. @@ -1328,7 +1350,7 @@ Basic customization .. versionchanged:: 3.7 ``object.__format__(x, '')`` is now equivalent to ``str(x)`` rather - than ``format(str(self), '')``. + than ``format(str(x), '')``. .. _richcmpfuncs: @@ -1438,8 +1460,8 @@ Basic customization .. note:: - By default, the :meth:`__hash__` values of str, bytes and datetime - objects are "salted" with an unpredictable random value. Although they + By default, the :meth:`__hash__` values of str and bytes objects are + "salted" with an unpredictable random value. Although they remain constant within an individual Python process, they are not predictable between repeated invocations of Python. @@ -1559,7 +1581,7 @@ not found on a module object through the normal lookup, i.e. the module ``__dict__`` before raising an :exc:`AttributeError`. If found, it is called with the attribute name and the result is returned. -The ``__dir__`` function should accept no arguments, and return a list of +The ``__dir__`` function should accept no arguments, and return a sequence of strings that represents the names accessible on module. If present, this function overrides the standard :func:`dir` search on a module. @@ -1611,21 +1633,32 @@ refers to the attribute whose name is the key of the property in the owner class' :attr:`~object.__dict__`. -.. method:: object.__get__(self, instance, owner) +.. method:: object.__get__(self, instance, owner=None) - Called to get the attribute of the owner class (class attribute access) or of an - instance of that class (instance attribute access). *owner* is always the owner - class, while *instance* is the instance that the attribute was accessed through, - or ``None`` when the attribute is accessed through the *owner*. This method - should return the (computed) attribute value or raise an :exc:`AttributeError` - exception. + Called to get the attribute of the owner class (class attribute access) or + of an instance of that class (instance attribute access). The optional + *owner* argument is the owner class, while *instance* is the instance that + the attribute was accessed through, or ``None`` when the attribute is + accessed through the *owner*. + This method should return the computed attribute value or raise an + :exc:`AttributeError` exception. + + :PEP:`252` specifies that :meth:`__get__` is callable with one or two + arguments. Python's own built-in descriptors support this specification; + however, it is likely that some third-party tools have descriptors + that require both arguments. Python's own :meth:`__getattribute__` + implementation always passes in both arguments whether they are required + or not. .. method:: object.__set__(self, instance, value) Called to set the attribute on an instance *instance* of the owner class to a new value, *value*. + Note, adding :meth:`__set__` or :meth:`__delete__` changes the kind of + descriptor to a "data descriptor". See :ref:`descriptor-invocation` for + more details. .. method:: object.__delete__(self, instance) @@ -1637,8 +1670,22 @@ class' :attr:`~object.__dict__`. Called at the time the owning class *owner* is created. The descriptor has been assigned to *name*. - .. versionadded:: 3.6 + .. note:: + :meth:`__set_name__` is only called implicitly as part of the + :class:`type` constructor, so it will need to be called explicitly with + the appropriate parameters when a descriptor is added to a class after + initial creation:: + + class A: + pass + descr = custom_descriptor() + A.attr = descr + descr.__set_name__(A, 'attr') + + See :ref:`class-object-creation` for more details. + + .. versionadded:: 3.6 The attribute :attr:`__objclass__` is interpreted by the :mod:`inspect` module as specifying the class where this object was defined (setting this @@ -1698,7 +1745,7 @@ the descriptor defines :meth:`__set__` and/or :meth:`__delete__`, it is a data descriptor; if it defines neither, it is a non-data descriptor. Normally, data descriptors define both :meth:`__get__` and :meth:`__set__`, while non-data descriptors have just the :meth:`__get__` method. Data descriptors with -:meth:`__set__` and :meth:`__get__` defined always override a redefinition in an +:meth:`__get__` and :meth:`__set__` (and/or :meth:`__delete__`) defined always override a redefinition in an instance dictionary. In contrast, non-data descriptors can be overridden by instances. @@ -1779,6 +1826,10 @@ Notes on using *__slots__* (the other bases must have empty slot layouts) - violations raise :exc:`TypeError`. +* If an iterator is used for *__slots__* then a descriptor is created for each + of the iterator's values. However, the *__slots__* attribute will be an empty + iterator. + .. _class-customization: Customizing class creation @@ -1804,7 +1855,7 @@ class defining the method. class, as in:: class Philosopher: - def __init_subclass__(cls, default_name, **kwargs): + def __init_subclass__(cls, /, default_name, **kwargs): super().__init_subclass__(**kwargs) cls.default_name = default_name @@ -1909,7 +1960,10 @@ Preparing the class namespace Once the appropriate metaclass has been identified, then the class namespace is prepared. If the metaclass has a ``__prepare__`` attribute, it is called as ``namespace = metaclass.__prepare__(name, bases, **kwds)`` (where the -additional keyword arguments, if any, come from the class definition). +additional keyword arguments, if any, come from the class definition). The +``__prepare__`` method should be implemented as a :func:`classmethod`. The +namespace returned by ``__prepare__`` is passed in to ``__new__``, but when +the final class object is created the namespace is copied into a new ``dict``. If the metaclass has no ``__prepare__`` attribute, then the class namespace is initialised as an empty ordered mapping. @@ -2110,8 +2164,8 @@ operators. It is recommended that both mappings and sequences implement the mappings, ``in`` should search the mapping's keys; for sequences, it should search through the values. It is further recommended that both mappings and sequences implement the :meth:`__iter__` method to allow efficient iteration -through the container; for mappings, :meth:`__iter__` should be the same as -:meth:`keys`; for sequences, it should iterate through the values. +through the container; for mappings, :meth:`__iter__` should iterate +through the object's keys; for sequences, it should iterate through the values. .. method:: object.__len__(self) @@ -2137,7 +2191,9 @@ through the container; for mappings, :meth:`__iter__` should be the same as Called to implement :func:`operator.length_hint`. Should return an estimated length for the object (which may be greater or less than the actual length). - The length must be an integer ``>=`` 0. This method is purely an + The length must be an integer ``>=`` 0. The return value may also be + :const:`NotImplemented`, which is treated the same as if the + ``__length_hint__`` method didn't exist at all. This method is purely an optimization and is never required for correctness. .. versionadded:: 3.4 @@ -2223,9 +2279,9 @@ through the container; for mappings, :meth:`__iter__` should be the same as The membership test operators (:keyword:`in` and :keyword:`not in`) are normally -implemented as an iteration through a sequence. However, container objects can +implemented as an iteration through a container. However, container objects can supply the following special method with a more efficient implementation, which -also does not require the object be a sequence. +also does not require the object be iterable. .. method:: object.__contains__(self, item) @@ -2293,7 +2349,7 @@ left undefined. object.__rfloordiv__(self, other) object.__rmod__(self, other) object.__rdivmod__(self, other) - object.__rpow__(self, other) + object.__rpow__(self, other[, modulo]) object.__rlshift__(self, other) object.__rrshift__(self, other) object.__rand__(self, other) @@ -2387,11 +2443,9 @@ left undefined. functions). Presence of this method indicates that the numeric object is an integer type. Must return an integer. - .. note:: - - In order to have a coherent integer type class, when :meth:`__index__` is - defined :meth:`__int__` should also be defined, and both should return - the same value. + If :meth:`__int__`, :meth:`__float__` and :meth:`__complex__` are not + defined then corresponding built-in functions :func:`int`, :func:`float` + and :func:`complex` fall back to :meth:`__index__`. .. method:: object.__round__(self, [,ndigits]) @@ -2547,7 +2601,7 @@ Awaitable Objects ----------------- An :term:`awaitable` object generally implements an :meth:`__await__` method. -:term:`Coroutine` objects returned from :keyword:`async def` functions +:term:`Coroutine objects ` returned from :keyword:`async def` functions are awaitable. .. note:: @@ -2572,7 +2626,7 @@ are awaitable. Coroutine Objects ----------------- -:term:`Coroutine` objects are :term:`awaitable` objects. +:term:`Coroutine objects ` are :term:`awaitable` objects. A coroutine's execution can be controlled by calling :meth:`__await__` and iterating over the result. When the coroutine has finished executing and returns, the iterator raises :exc:`StopIteration`, and the exception's @@ -2680,13 +2734,13 @@ Asynchronous context managers can be used in an :keyword:`async with` statement. .. method:: object.__aenter__(self) - This method is semantically similar to the :meth:`__enter__`, with only - difference that it must return an *awaitable*. + Semantically similar to :meth:`__enter__`, the only + difference being that it must return an *awaitable*. .. method:: object.__aexit__(self, exc_type, exc_value, traceback) - This method is semantically similar to the :meth:`__exit__`, with only - difference that it must return an *awaitable*. + Semantically similar to :meth:`__exit__`, the only + difference being that it must return an *awaitable*. An example of an asynchronous context manager class:: diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index ba7130d6362163..55ac01b6a844dc 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -22,7 +22,9 @@ The following are blocks: a module, a function body, and a class definition. Each command typed interactively is a block. A script file (a file given as standard input to the interpreter or specified as a command line argument to the interpreter) is a code block. A script command (a command specified on the -interpreter command line with the :option:`-c` option) is a code block. The string +interpreter command line with the :option:`-c` option) is a code block. +A module run as a top level script (as module ``__main__``) from the command +line using a :option:`-m` argument is also a code block. The string argument passed to the built-in functions :func:`eval` and :func:`exec` is a code block. @@ -243,7 +245,7 @@ re-entering the offending piece of code from the top). When an exception is not handled at all, the interpreter terminates execution of the program, or returns to its interactive main loop. In either case, it prints -a stack backtrace, except when the exception is :exc:`SystemExit`. +a stack traceback, except when the exception is :exc:`SystemExit`. Exceptions are identified by class instances. The :keyword:`except` clause is selected depending on the class of the instance: it must reference the class of diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index cf7d05eef6746a..b68c29860cf332 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -13,7 +13,7 @@ This chapter explains the meaning of the elements of expressions in Python. be used to describe syntax, not lexical analysis. When (one alternative of) a syntax rule has the form -.. productionlist:: * +.. productionlist:: python-grammar name: `othername` and no semantics are given, the semantics of this form of ``name`` are the same @@ -28,7 +28,7 @@ Arithmetic conversions .. index:: pair: arithmetic; conversion When a description of an arithmetic operator below uses the phrase "the numeric -arguments are converted to a common type," this means that the operator +arguments are converted to a common type", this means that the operator implementation for built-in types works as follows: * If either argument is a complex number, the other is converted to complex; @@ -54,7 +54,7 @@ Atoms are the most basic elements of expressions. The simplest atoms are identifiers or literals. Forms enclosed in parentheses, brackets or braces are also categorized syntactically as atoms. The syntax for atoms is: -.. productionlist:: +.. productionlist:: python-grammar atom: `identifier` | `literal` | `enclosure` enclosure: `parenth_form` | `list_display` | `dict_display` | `set_display` : | `generator_expression` | `yield_atom` @@ -103,7 +103,7 @@ Literals Python supports string and bytes literals and various numeric literals: -.. productionlist:: +.. productionlist:: python-grammar literal: `stringliteral` | `bytesliteral` : | `integer` | `floatnumber` | `imagnumber` @@ -134,7 +134,7 @@ Parenthesized forms A parenthesized form is an optional expression list enclosed in parentheses: -.. productionlist:: +.. productionlist:: python-grammar parenth_form: "(" [`starred_expression`] ")" A parenthesized expression list yields whatever that expression list yields: if @@ -144,13 +144,12 @@ the single expression that makes up the expression list. .. index:: pair: empty; tuple An empty pair of parentheses yields an empty tuple object. Since tuples are -immutable, the rules for literals apply (i.e., two occurrences of the empty +immutable, the same rules as for literals apply (i.e., two occurrences of the empty tuple may or may not yield the same object). .. index:: - single: comma; tuple display - pair: tuple; display - single: , (comma); tuple display + single: comma + single: , (comma) Note that tuples are not formed by the parentheses, but rather by use of the comma operator. The exception is the empty tuple, for which parentheses *are* @@ -178,8 +177,8 @@ called "displays", each of them in two flavors: Common syntax elements for comprehensions are: -.. productionlist:: - comprehension: `expression` `comp_for` +.. productionlist:: python-grammar + comprehension: `assignment_expression` `comp_for` comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`] comp_iter: `comp_for` | `comp_if` comp_if: "if" `expression_nocond` [`comp_iter`] @@ -196,7 +195,7 @@ the comprehension is executed in a separate implicitly nested scope. This ensure that names assigned to in the target list don't "leak" into the enclosing scope. The iterable expression in the leftmost :keyword:`!for` clause is evaluated -directly in the enclosing scope and then passed as an argument to the implictly +directly in the enclosing scope and then passed as an argument to the implicitly nested scope. Subsequent :keyword:`!for` clauses and any filter condition in the leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as they may depend on the values obtained from the leftmost iterable. For example: @@ -244,7 +243,7 @@ List displays A list display is a possibly empty series of expressions enclosed in square brackets: -.. productionlist:: +.. productionlist:: python-grammar list_display: "[" [`starred_list` | `comprehension`] "]" A list display yields a new list object, the contents being specified by either @@ -268,7 +267,7 @@ Set displays A set display is denoted by curly braces and distinguishable from dictionary displays by the lack of colons separating keys and values: -.. productionlist:: +.. productionlist:: python-grammar set_display: "{" (`starred_list` | `comprehension`) "}" A set display yields a new mutable set object, the contents being specified by @@ -297,7 +296,7 @@ Dictionary displays A dictionary display is a possibly empty series of key/datum pairs enclosed in curly braces: -.. productionlist:: +.. productionlist:: python-grammar dict_display: "{" [`key_datum_list` | `dict_comprehension`] "}" key_datum_list: `key_datum` ("," `key_datum`)* [","] key_datum: `expression` ":" `expression` | "**" `or_expr` @@ -337,6 +336,12 @@ all mutable objects.) Clashes between duplicate keys are not detected; the last datum (textually rightmost in the display) stored for a given key value prevails. +.. versionchanged:: 3.8 + Prior to Python 3.8, in dict comprehensions, the evaluation order of key + and value was not well-defined. In CPython, the value was evaluated before + the key. Starting with 3.8, the key is evaluated before the value, as + proposed by :pep:`572`. + .. _genexpr: @@ -350,7 +355,7 @@ Generator expressions A generator expression is a compact generator notation in parentheses: -.. productionlist:: +.. productionlist:: python-grammar generator_expression: "(" `expression` `comp_for` ")" A generator expression yields a new generator object. Its syntax is the same as @@ -404,7 +409,7 @@ Yield expressions pair: yield; expression pair: generator; function -.. productionlist:: +.. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" yield_expression: "yield" [`expression_list` | "from" `expression`] @@ -479,8 +484,8 @@ will raise :exc:`AttributeError` or :exc:`TypeError`, while When the underlying iterator is complete, the :attr:`~StopIteration.value` attribute of the raised :exc:`StopIteration` instance becomes the value of the yield expression. It can be either set explicitly when raising -:exc:`StopIteration`, or automatically when the sub-iterator is a generator -(by returning a value from the sub-generator). +:exc:`StopIteration`, or automatically when the subiterator is a generator +(by returning a value from the subgenerator). .. versionchanged:: 3.3 Added ``yield from `` to delegate control flow to a subiterator. @@ -499,7 +504,7 @@ on the right hand side of an assignment statement. :pep:`380` - Syntax for Delegating to a Subgenerator The proposal to introduce the :token:`yield_from` syntax, making delegation - to sub-generators easy. + to subgenerators easy. :pep:`525` - Asynchronous Generators The proposal that expanded on :pep:`492` by adding generator capabilities to @@ -608,7 +613,7 @@ Asynchronous generator functions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The presence of a yield expression in a function or method defined using -:keyword:`async def` further defines the function as a +:keyword:`async def` further defines the function as an :term:`asynchronous generator` function. When an asynchronous generator function is called, it returns an @@ -673,13 +678,13 @@ which are used to control the execution of a generator function. Returns an awaitable which when run starts to execute the asynchronous generator or resumes it at the last executed yield expression. When an - asynchronous generator function is resumed with a :meth:`~agen.__anext__` + asynchronous generator function is resumed with an :meth:`~agen.__anext__` method, the current yield expression always evaluates to :const:`None` in the returned awaitable, which when run will continue to the next yield expression. The value of the :token:`expression_list` of the yield expression is the value of the :exc:`StopIteration` exception raised by the completing coroutine. If the asynchronous generator exits without - yielding another value, the awaitable instead raises an + yielding another value, the awaitable instead raises a :exc:`StopAsyncIteration` exception, signalling that the asynchronous iteration has completed. @@ -707,7 +712,7 @@ which are used to control the execution of a generator function. where the asynchronous generator was paused, and returns the next value yielded by the generator function as the value of the raised :exc:`StopIteration` exception. If the asynchronous generator exits - without yielding another value, an :exc:`StopAsyncIteration` exception is + without yielding another value, a :exc:`StopAsyncIteration` exception is raised by the awaitable. If the generator function does not catch the passed-in exception, or raises a different exception, then when the awaitable is run that exception @@ -741,7 +746,7 @@ Primaries Primaries represent the most tightly bound operations of the language. Their syntax is: -.. productionlist:: +.. productionlist:: python-grammar primary: `atom` | `attributeref` | `subscription` | `slicing` | `call` @@ -756,7 +761,7 @@ Attribute references An attribute reference is a primary followed by a period and a name: -.. productionlist:: +.. productionlist:: python-grammar attributeref: `primary` "." `identifier` .. index:: @@ -794,7 +799,7 @@ Subscriptions A subscription selects an item of a sequence (string, tuple or list) or mapping (dictionary) object: -.. productionlist:: +.. productionlist:: python-grammar subscription: `primary` "[" `expression_list` "]" The primary must evaluate to an object that supports subscription (lists or @@ -850,7 +855,7 @@ A slicing selects a range of items in a sequence object (e.g., a string, tuple or list). Slicings may be used as expressions or as targets in assignment or :keyword:`del` statements. The syntax for a slicing: -.. productionlist:: +.. productionlist:: python-grammar slicing: `primary` "[" `slice_list` "]" slice_list: `slice_item` ("," `slice_item`)* [","] slice_item: `expression` | `proper_slice` @@ -900,13 +905,14 @@ Calls A call calls a callable object (e.g., a :term:`function`) with a possibly empty series of :term:`arguments `: -.. productionlist:: +.. productionlist:: python-grammar call: `primary` "(" [`argument_list` [","] | `comprehension`] ")" argument_list: `positional_arguments` ["," `starred_and_keywords`] : ["," `keywords_arguments`] : | `starred_and_keywords` ["," `keywords_arguments`] : | `keywords_arguments` - positional_arguments: ["*"] `expression` ("," ["*"] `expression`)* + positional_arguments: positional_item ("," positional_item)* + positional_item: `assignment_expression` | "*" `expression` starred_and_keywords: ("*" `expression` | `keyword_item`) : ("," "*" `expression` | "," `keyword_item`)* keywords_arguments: (`keyword_item` | "**" `expression`) @@ -1082,7 +1088,7 @@ Await expression Suspend the execution of :term:`coroutine` on an :term:`awaitable` object. Can only be used inside a :term:`coroutine function`. -.. productionlist:: +.. productionlist:: python-grammar await_expr: "await" `primary` .. versionadded:: 3.5 @@ -1100,7 +1106,7 @@ The power operator The power operator binds more tightly than unary operators on its left; it binds less tightly than unary operators on its right. The syntax is: -.. productionlist:: +.. productionlist:: python-grammar power: (`await_expr` | `primary`) ["**" `u_expr`] Thus, in an unparenthesized sequence of power and unary operators, the operators @@ -1133,7 +1139,7 @@ Unary arithmetic and bitwise operations All unary arithmetic and bitwise operations have the same priority: -.. productionlist:: +.. productionlist:: python-grammar u_expr: `power` | "-" `u_expr` | "+" `u_expr` | "~" `u_expr` .. index:: @@ -1177,7 +1183,7 @@ that some of these operations also apply to certain non-numeric types. Apart from the power operator, there are only two levels, one for multiplicative operators and one for additive operators: -.. productionlist:: +.. productionlist:: python-grammar m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "@" `m_expr` | : `m_expr` "//" `u_expr` | `m_expr` "/" `u_expr` | : `m_expr` "%" `u_expr` @@ -1273,7 +1279,7 @@ Shifting operations The shifting operations have lower priority than the arithmetic operations: -.. productionlist:: +.. productionlist:: python-grammar shift_expr: `a_expr` | `shift_expr` ("<<" | ">>") `a_expr` These operators accept integers as arguments. They shift the first argument to @@ -1294,7 +1300,7 @@ Binary bitwise operations Each of the three bitwise operations has a different priority level: -.. productionlist:: +.. productionlist:: python-grammar and_expr: `shift_expr` | `and_expr` "&" `shift_expr` xor_expr: `and_expr` | `xor_expr` "^" `and_expr` or_expr: `xor_expr` | `or_expr` "|" `xor_expr` @@ -1343,7 +1349,7 @@ lower than that of any arithmetic, shifting or bitwise operation. Also unlike C, expressions like ``a < b < c`` have the interpretation that is conventional in mathematics: -.. productionlist:: +.. productionlist:: python-grammar comparison: `or_expr` (`comp_operator` `or_expr`)* comp_operator: "<" | ">" | "==" | ">=" | "<=" | "!=" : | "is" ["not"] | ["not"] "in" @@ -1416,8 +1422,13 @@ built-in types. The not-a-number values ``float('NaN')`` and ``decimal.Decimal('NaN')`` are special. Any ordered comparison of a number to a not-a-number value is false. A counter-intuitive implication is that not-a-number values are not equal to - themselves. For example, if ``x = float('NaN')``, ``3 < x``, ``x < 3``, ``x - == x``, ``x != x`` are all false. This behavior is compliant with IEEE 754. + themselves. For example, if ``x = float('NaN')``, ``3 < x``, ``x < 3`` and + ``x == x`` are all false, while ``x != x`` is true. This behavior is + compliant with IEEE 754. + +* ``None`` and ``NotImplemented`` are singletons. :PEP:`8` advises that + comparisons for singletons should always be done with ``is`` or ``is not``, + never the equality operators. * Binary sequences (instances of :class:`bytes` or :class:`bytearray`) can be compared within and across their types. They compare lexicographically using @@ -1436,25 +1447,9 @@ built-in types. :exc:`TypeError`. Sequences compare lexicographically using comparison of corresponding - elements, whereby reflexivity of the elements is enforced. - - In enforcing reflexivity of elements, the comparison of collections assumes - that for a collection element ``x``, ``x == x`` is always true. Based on - that assumption, element identity is compared first, and element comparison - is performed only for distinct elements. This approach yields the same - result as a strict element comparison would, if the compared elements are - reflexive. For non-reflexive elements, the result is different than for - strict element comparison, and may be surprising: The non-reflexive - not-a-number values for example result in the following comparison behavior - when used in a list:: - - >>> nan = float('NaN') - >>> nan is nan - True - >>> nan == nan - False <-- the defined non-reflexive behavior of NaN - >>> [nan] == [nan] - True <-- list enforces reflexivity and tests identity first + elements. The built-in containers typically assume identical objects are + equal to themselves. That lets them bypass equality tests for identical + objects to improve performance and to maintain their internal invariants. Lexicographical comparison between built-in collections works as follows: @@ -1563,14 +1558,15 @@ y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and ``False`` otherwise. For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is -produced while iterating over ``y``. If an exception is raised during the -iteration, it is as if :keyword:`in` raised that exception. +:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z``, for which the +expression ``x is z or x == z`` is true, is produced while iterating over ``y``. +If an exception is raised during the iteration, it is as if :keyword:`in` raised +that exception. Lastly, the old-style iteration protocol is tried: if a class defines :meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative -integer index *i* such that ``x == y[i]``, and all lower integer indices do not -raise :exc:`IndexError` exception. (If any other exception is raised, it is as +integer index *i* such that ``x is y[i] or x == y[i]``, and no lower integer index +raises the :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). .. index:: @@ -1579,7 +1575,7 @@ if :keyword:`in` raised that exception). pair: membership; test object: sequence -The operator :keyword:`not in` is defined to have the inverse true value of +The operator :keyword:`not in` is defined to have the inverse truth value of :keyword:`in`. .. index:: @@ -1594,8 +1590,8 @@ The operator :keyword:`not in` is defined to have the inverse true value of Identity comparisons -------------------- -The operators :keyword:`is` and :keyword:`is not` test for object identity: ``x -is y`` is true if and only if *x* and *y* are the same object. Object identity +The operators :keyword:`is` and :keyword:`is not` test for an object's identity: ``x +is y`` is true if and only if *x* and *y* are the same object. An Object's identity is determined using the :meth:`id` function. ``x is not y`` yields the inverse truth value. [#]_ @@ -1612,7 +1608,7 @@ Boolean operations pair: Conditional; expression pair: Boolean; operation -.. productionlist:: +.. productionlist:: python-grammar or_test: `and_test` | `or_test` "or" `and_test` and_test: `not_test` | `and_test` "and" `not_test` not_test: `comparison` | "not" `not_test` @@ -1648,6 +1644,34 @@ returns a boolean value regardless of the type of its argument (for example, ``not 'foo'`` produces ``False`` rather than ``''``.) +Assignment expressions +====================== + +.. productionlist:: python-grammar + assignment_expression: [`identifier` ":="] `expression` + +An assignment expression (sometimes also called a "named expression" or +"walrus") assigns an :token:`expression` to an :token:`identifier`, while also +returning the value of the :token:`expression`. + +One common use case is when handling matched regular expressions: + +.. code-block:: python + + if matching := pattern.search(data): + do_something(matching) + +Or, when processing a file stream in chunks: + +.. code-block:: python + + while chunk := file.read(9000): + process(chunk) + +.. versionadded:: 3.8 + See :pep:`572` for more details about assignment expressions. + + .. _if_expr: Conditional expressions @@ -1659,7 +1683,7 @@ Conditional expressions single: if; conditional expression single: else; conditional expression -.. productionlist:: +.. productionlist:: python-grammar conditional_expression: `or_test` ["if" `or_test` "else" `expression`] expression: `conditional_expression` | `lambda_expr` expression_nocond: `or_test` | `lambda_expr_nocond` @@ -1686,7 +1710,7 @@ Lambdas pair: anonymous; function single: : (colon); lambda expression -.. productionlist:: +.. productionlist:: python-grammar lambda_expr: "lambda" [`parameter_list`] ":" `expression` lambda_expr_nocond: "lambda" [`parameter_list`] ":" `expression_nocond` @@ -1713,11 +1737,11 @@ Expression lists pair: expression; list single: , (comma); expression list -.. productionlist:: +.. productionlist:: python-grammar expression_list: `expression` ("," `expression`)* [","] starred_list: `starred_item` ("," `starred_item`)* [","] starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] - starred_item: `expression` | "*" `or_expr` + starred_item: `assignment_expression` | "*" `or_expr` .. index:: object: tuple @@ -1790,6 +1814,8 @@ precedence and have a left-to-right chaining feature as described in the +-----------------------------------------------+-------------------------------------+ | Operator | Description | +===============================================+=====================================+ +| ``:=`` | Assignment expression | ++-----------------------------------------------+-------------------------------------+ | :keyword:`lambda` | Lambda expression | +-----------------------------------------------+-------------------------------------+ | :keyword:`if ` -- :keyword:`!else` | Conditional expression | @@ -1827,7 +1853,8 @@ precedence and have a left-to-right chaining feature as described in the | ``x[index]``, ``x[index:index]``, | Subscription, slicing, | | ``x(arguments...)``, ``x.attribute`` | call, attribute reference | +-----------------------------------------------+-------------------------------------+ -| ``(expressions...)``, | Binding or tuple display, | +| ``(expressions...)``, | Binding or parenthesized | +| | expression, | | ``[expressions...]``, | list display, | | ``{key: value...}``, | dictionary display, | | ``{expressions...}`` | set display | diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index 83d0f8532c6366..acf83765b5796c 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -1,7 +1,19 @@ Full Grammar specification ========================== -This is the full Python grammar, as it is read by the parser generator and used -to parse Python source files: +This is the full Python grammar, derived directly from the grammar +used to generate the CPython parser (see :source:`Grammar/python.gram`). +The version here omits details related to code generation and +error recovery. -.. literalinclude:: ../../Grammar/Grammar +The notation is a mixture of `EBNF +`_ +and `PEG `_. +In particular, ``&`` followed by a symbol, token or parenthesized +group indicates a positive lookahead (i.e., is required to match but +not consumed), while ``!`` indicates a negative lookahead (i.e., is +required _not_ to match). We use the ``|`` separator to mean PEG's +"ordered choice" (written as ``/`` in traditional PEG grammars). + +.. literalinclude:: ../../Grammar/python.gram + :language: peg diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 9a0ab39d3b4a3e..4c36e15dc06063 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -83,7 +83,7 @@ module. Specifically, any module that contains a ``__path__`` attribute is considered a package. All modules have a name. Subpackage names are separated from their parent -package name by dots, akin to Python's standard attribute access syntax. Thus +package name by a dot, akin to Python's standard attribute access syntax. Thus you might have a module called :mod:`sys` and a package called :mod:`email`, which in turn has a subpackage called :mod:`email.mime` and a module within that subpackage called :mod:`email.mime.text`. @@ -202,6 +202,8 @@ named module, the two module objects will *not* be the same. By contrast, reinitialise the module contents by rerunning the module's code. +.. _finders-and-loaders: + Finders and loaders ------------------- @@ -345,12 +347,11 @@ of what happens during the loading portion of import:: _init_module_attrs(spec, module) if spec.loader is None: - if spec.submodule_search_locations is not None: - # namespace package - sys.modules[spec.name] = module - else: - # unsupported - raise ImportError + # unsupported + raise ImportError + if spec.origin is None and spec.submodule_search_locations is not None: + # namespace package + sys.modules[spec.name] = module elif not hasattr(spec.loader, 'exec_module'): module = spec.loader.load_module(spec.name) # Set __loader__ and __package__ if missing. @@ -683,7 +684,7 @@ Before Python loads cached bytecode from ``.pyc`` file, it checks whether the cache is up-to-date with the source ``.py`` file. By default, Python does this by storing the source's last-modified timestamp and size in the cache file when writing it. At runtime, the import system then validates the cache file by -checking the stored metadata in the cache file against at source's +checking the stored metadata in the cache file against the source's metadata. Python also supports "hash-based" cache files, which store a hash of the source @@ -850,13 +851,13 @@ In order to support imports of modules and initialized packages and also to contribute portions to namespace packages, path entry finders must implement the :meth:`~importlib.abc.PathEntryFinder.find_spec` method. -:meth:`~importlib.abc.PathEntryFinder.find_spec` takes two argument, the +:meth:`~importlib.abc.PathEntryFinder.find_spec` takes two arguments: the fully qualified name of the module being imported, and the (optional) target module. ``find_spec()`` returns a fully populated spec for the module. This spec will always have "loader" set (with one exception). To indicate to the import machinery that the spec represents a namespace -:term:`portion`. the path entry finder sets "loader" on the spec to +:term:`portion`, the path entry finder sets "loader" on the spec to ``None`` and "submodule_search_locations" to a list containing the portion. @@ -914,13 +915,53 @@ the builtin :func:`__import__` function may be sufficient. This technique may also be employed at the module level to only alter the behaviour of import statements within that module. -To selectively prevent import of some modules from a hook early on the +To selectively prevent the import of some modules from a hook early on the meta path (rather than disabling the standard import system entirely), it is sufficient to raise :exc:`ModuleNotFoundError` directly from :meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning ``None``. The latter indicates that the meta path search should continue, while raising an exception terminates it immediately. +.. _relativeimports: + +Package Relative Imports +======================== + +Relative imports use leading dots. A single leading dot indicates a relative +import, starting with the current package. Two or more leading dots indicate a +relative import to the parent(s) of the current package, one level per dot +after the first. For example, given the following package layout:: + + package/ + __init__.py + subpackage1/ + __init__.py + moduleX.py + moduleY.py + subpackage2/ + __init__.py + moduleZ.py + moduleA.py + +In either ``subpackage1/moduleX.py`` or ``subpackage1/__init__.py``, +the following are valid relative imports:: + + from .moduleY import spam + from .moduleY import spam as ham + from . import moduleY + from ..subpackage1 import moduleY + from ..subpackage2.moduleZ import eggs + from ..moduleA import foo + +Absolute imports may use either the ``import <>`` or ``from <> import <>`` +syntax, but relative imports may only use the second form; the reason +for this is that:: + + import XXX.YYY.ZZZ + +should expose ``XXX.YYY.ZZZ`` as a usable expression, but .moduleY is +not a valid expression. + Special considerations for __main__ =================================== diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index bb7e3906dba607..72e874ee98e466 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -93,7 +93,7 @@ Notation The descriptions of lexical analysis and syntax use a modified BNF grammar notation. This uses the following style of definition: -.. productionlist:: * +.. productionlist:: notation name: `lc_letter` (`lc_letter` | "_")* lc_letter: "a"..."z" diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index fb04ccc839aaa3..77e0578f5d89b6 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -70,7 +70,7 @@ Comments A comment starts with a hash character (``#``) that is not part of a string literal, and ends at the end of the physical line. A comment signifies the end of the logical line unless the implicit line joining rules are invoked. Comments -are ignored by the syntax; they are not tokens. +are ignored by the syntax. .. _encodings: @@ -296,7 +296,7 @@ Unicode Character Database as included in the :mod:`unicodedata` module. Identifiers are unlimited in length. Case is significant. -.. productionlist:: +.. productionlist:: python-grammar identifier: `xid_start` `xid_continue`* id_start: id_continue: @@ -316,7 +316,7 @@ The Unicode category codes mentioned above stand for: * *Nd* - decimal numbers * *Pc* - connector punctuations * *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards + `_ to support backwards compatibility * *Other_ID_Continue* - likewise @@ -325,7 +325,7 @@ of identifiers is based on NFKC. A non-normative HTML file listing all valid identifier characters for Unicode 4.1 can be found at -https://www.dcl.hpi.uni-potsdam.de/home/loewis/table-3131.html. +https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt .. _keywords: @@ -376,11 +376,11 @@ characters: information on this convention. ``__*__`` - System-defined names. These names are defined by the interpreter and its - implementation (including the standard library). Current system names are - discussed in the :ref:`specialnames` section and elsewhere. More will likely - be defined in future versions of Python. *Any* use of ``__*__`` names, in - any context, that does not follow explicitly documented use, is subject to + System-defined names, informally known as "dunder" names. These names are + defined by the interpreter and its implementation (including the standard library). + Current system names are discussed in the :ref:`specialnames` section and elsewhere. + More will likely be defined in future versions of Python. *Any* use of ``__*__`` names, + in any context, that does not follow explicitly documented use, is subject to breakage without warning. ``__*`` @@ -412,7 +412,7 @@ String and Bytes literals String literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar stringliteral: [`stringprefix`](`shortstring` | `longstring`) stringprefix: "r" | "u" | "R" | "U" | "f" | "F" : | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" @@ -424,7 +424,7 @@ String literals are described by the following lexical definitions: longstringchar: stringescapeseq: "\" -.. productionlist:: +.. productionlist:: python-grammar bytesliteral: `bytesprefix`(`shortbytes` | `longbytes`) bytesprefix: "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" shortbytes: "'" `shortbytesitem`* "'" | '"' `shortbytesitem`* '"' @@ -594,11 +594,9 @@ escape sequences only recognized in string literals fall into the category of unrecognized escapes for bytes literals. .. versionchanged:: 3.6 - Unrecognized escape sequences produce a :exc:`DeprecationWarning`. - - .. versionchanged:: 3.8 - Unrecognized escape sequences produce a :exc:`SyntaxWarning`. In - some future version of Python they will be a :exc:`SyntaxError`. + Unrecognized escape sequences produce a :exc:`DeprecationWarning`. In + a future Python version they will be a :exc:`SyntaxWarning` and + eventually a :exc:`SyntaxError`. Even in a raw literal, quotes can be escaped with a backslash, but the backslash remains in the result; for example, ``r"\""`` is a valid string @@ -639,9 +637,11 @@ and formatted string literals may be concatenated with plain string literals. single: string; formatted literal single: string; interpolated literal single: f-string + single: fstring single: {} (curly brackets); in formatted string literal single: ! (exclamation); in formatted string literal single: : (colon); in formatted string literal + single: = (equals); for help in debugging using string literals .. _f-strings: Formatted string literals @@ -659,9 +659,9 @@ Escape sequences are decoded like in ordinary string literals (except when a literal is also marked as a raw string). After decoding, the grammar for the contents of the string is: -.. productionlist:: +.. productionlist:: python-grammar f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* - replacement_field: "{" `f_expression` ["!" `conversion`] [":" `format_spec`] "}" + replacement_field: "{" `f_expression` ["="] ["!" `conversion`] [":" `format_spec`] "}" f_expression: (`conditional_expression` | "*" `or_expr`) : ("," `conditional_expression` | "," "*" `or_expr`)* [","] : | `yield_expression` @@ -673,18 +673,36 @@ The parts of the string outside curly braces are treated literally, except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced with the corresponding single curly brace. A single opening curly bracket ``'{'`` marks a replacement field, which starts with a -Python expression. After the expression, there may be a conversion field, -introduced by an exclamation point ``'!'``. A format specifier may also -be appended, introduced by a colon ``':'``. A replacement field ends -with a closing curly bracket ``'}'``. +Python expression. To display both the expression text and its value after +evaluation, (useful in debugging), an equal sign ``'='`` may be added after the +expression. A conversion field, introduced by an exclamation point ``'!'`` may +follow. A format specifier may also be appended, introduced by a colon ``':'``. +A replacement field ends with a closing curly bracket ``'}'``. Expressions in formatted string literals are treated like regular Python expressions surrounded by parentheses, with a few exceptions. -An empty expression is not allowed, and a :keyword:`lambda` expression -must be surrounded by explicit parentheses. Replacement expressions -can contain line breaks (e.g. in triple-quoted strings), but they -cannot contain comments. Each expression is evaluated in the context -where the formatted string literal appears, in order from left to right. +An empty expression is not allowed, and both :keyword:`lambda` and +assignment expressions ``:=`` must be surrounded by explicit parentheses. +Replacement expressions can contain line breaks (e.g. in triple-quoted +strings), but they cannot contain comments. Each expression is evaluated +in the context where the formatted string literal appears, in order from +left to right. + +.. versionchanged:: 3.7 + Prior to Python 3.7, an :keyword:`await` expression and comprehensions + containing an :keyword:`async for` clause were illegal in the expressions + in formatted string literals due to a problem with the implementation. + +When the equal sign ``'='`` is provided, the output will have the expression +text, the ``'='`` and the evaluated value. Spaces after the opening brace +``'{'``, within the expression and after the ``'='`` are all retained in the +output. By default, the ``'='`` causes the :func:`repr` of the expression to be +provided, unless there is a format specified. When a format is specified it +defaults to the :func:`str` of the expression unless a conversion ``'!r'`` is +declared. + +.. versionadded:: 3.8 + The equal sign ``'='``. If a conversion is specified, the result of evaluating the expression is converted before formatting. Conversion ``'!s'`` calls :func:`str` on @@ -700,7 +718,7 @@ Top-level format specifiers may include nested replacement fields. These nested fields may include their own conversion fields and :ref:`format specifiers `, but may not include more deeply-nested replacement fields. The :ref:`format specifier mini-language ` is the same as that used by -the string .format() method. +the :meth:`str.format` method. Formatted string literals may be concatenated, but replacement fields cannot be split across literals. @@ -720,9 +738,22 @@ Some examples of formatted string literals:: >>> today = datetime(year=2017, month=1, day=27) >>> f"{today:%B %d, %Y}" # using date format specifier 'January 27, 2017' + >>> f"{today=:%B %d, %Y}" # using date format specifier and debugging + 'today=January 27, 2017' >>> number = 1024 >>> f"{number:#0x}" # using integer format specifier '0x400' + >>> foo = "bar" + >>> f"{ foo = }" # preserves whitespace + " foo = 'bar'" + >>> line = "The mill's closed" + >>> f"{line = }" + 'line = "The mill\'s closed"' + >>> f"{line = :20}" + "line = The mill's closed " + >>> f"{line = !r:20}" + 'line = "The mill\'s closed" ' + A consequence of sharing the same syntax as regular string literals is that characters in the replacement fields must not conflict with the @@ -789,7 +820,7 @@ Integer literals Integer literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar integer: `decinteger` | `bininteger` | `octinteger` | `hexinteger` decinteger: `nonzerodigit` (["_"] `digit`)* | "0"+ (["_"] "0")* bininteger: "0" ("b" | "B") (["_"] `bindigit`)+ @@ -833,7 +864,7 @@ Floating point literals Floating point literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar floatnumber: `pointfloat` | `exponentfloat` pointfloat: [`digitpart`] `fraction` | `digitpart` "." exponentfloat: (`digitpart` | `pointfloat`) `exponent` @@ -863,7 +894,7 @@ Imaginary literals Imaginary literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar imagnumber: (`floatnumber` | `digitpart`) ("j" | "J") An imaginary literal yields a complex number with a real part of 0.0. Complex @@ -888,7 +919,7 @@ The following tokens are operators: + - * ** / // % @ - << >> & | ^ ~ + << >> & | ^ ~ := < > <= >= == != @@ -930,4 +961,4 @@ occurrence outside string literals and comments is an unconditional error: .. rubric:: Footnotes -.. [#] http://www.unicode.org/Public/11.0.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/11.0.0/ucd/NameAliases.txt diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 00964afc6d3d2f..93be32713ff32a 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -11,7 +11,7 @@ A simple statement is comprised within a single logical line. Several simple statements may occur on a single line separated by semicolons. The syntax for simple statements is: -.. productionlist:: +.. productionlist:: python-grammar simple_stmt: `expression_stmt` : | `assert_stmt` : | `assignment_stmt` @@ -46,7 +46,7 @@ result; in Python, procedures return the value ``None``). Other uses of expression statements are allowed and occasionally useful. The syntax for an expression statement is: -.. productionlist:: +.. productionlist:: python-grammar expression_stmt: `starred_expression` An expression statement evaluates the expression list (which may be a single @@ -82,7 +82,7 @@ Assignment statements Assignment statements are used to (re)bind names to values and to modify attributes or items of mutable objects: -.. productionlist:: +.. productionlist:: python-grammar assignment_stmt: (`target_list` "=")+ (`starred_expression` | `yield_expression`) target_list: `target` ("," `target`)* [","] target: `identifier` @@ -169,12 +169,12 @@ Assignment of an object to a single target is recursively defined as follows. .. _attr-target-note: Note: If the object is a class instance and the attribute reference occurs on - both sides of the assignment operator, the RHS expression, ``a.x`` can access + both sides of the assignment operator, the right-hand side expression, ``a.x`` can access either an instance attribute or (if no instance attribute exists) a class - attribute. The LHS target ``a.x`` is always set as an instance attribute, + attribute. The left-hand side target ``a.x`` is always set as an instance attribute, creating it if necessary. Thus, the two occurrences of ``a.x`` do not - necessarily refer to the same attribute: if the RHS expression refers to a - class attribute, the LHS creates a new instance attribute as the target of the + necessarily refer to the same attribute: if the right-hand side expression refers to a + class attribute, the left-hand side creates a new instance attribute as the target of the assignment:: class Cls: @@ -280,7 +280,7 @@ Augmented assignment statements Augmented assignment is the combination, in a single statement, of a binary operation and an assignment statement: -.. productionlist:: +.. productionlist:: python-grammar augmented_assignment_stmt: `augtarget` `augop` (`expression_list` | `yield_expression`) augtarget: `identifier` | `attributeref` | `subscription` | `slicing` augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" @@ -328,11 +328,11 @@ Annotated assignment statements :term:`Annotation ` assignment is the combination, in a single statement, of a variable or attribute annotation and an optional assignment statement: -.. productionlist:: - annotated_assignment_stmt: `augtarget` ":" `expression` ["=" `expression`] +.. productionlist:: python-grammar + annotated_assignment_stmt: `augtarget` ":" `expression` + : ["=" (`starred_expression` | `yield_expression`)] -The difference from normal :ref:`assignment` is that only single target and -only single right hand side value is allowed. +The difference from normal :ref:`assignment` is that only single target is allowed. For simple names as assignment targets, if in class or module scope, the annotations are evaluated and stored in a special class or module @@ -366,6 +366,11 @@ target, then the interpreter evaluates the target except for the last syntax for type annotations that can be used in static analysis tools and IDEs. +.. versionchanged:: 3.8 + Now annotated assignments allow same expressions in the right hand side as + the regular assignments. Previously, some expressions (like un-parenthesized + tuple expressions) caused a syntax error. + .. _assert: @@ -380,7 +385,7 @@ The :keyword:`!assert` statement Assert statements are a convenient way to insert debugging assertions into a program: -.. productionlist:: +.. productionlist:: python-grammar assert_stmt: "assert" `expression` ["," `expression`] The simple form, ``assert expression``, is equivalent to :: @@ -420,7 +425,7 @@ The :keyword:`!pass` statement pair: null; operation pair: null; operation -.. productionlist:: +.. productionlist:: python-grammar pass_stmt: "pass" :keyword:`pass` is a null operation --- when it is executed, nothing happens. @@ -442,7 +447,7 @@ The :keyword:`!del` statement pair: deletion; target triple: deletion; target; list -.. productionlist:: +.. productionlist:: python-grammar del_stmt: "del" `target_list` Deletion is recursively defined very similar to the way assignment is defined. @@ -481,7 +486,7 @@ The :keyword:`!return` statement pair: function; definition pair: class; definition -.. productionlist:: +.. productionlist:: python-grammar return_stmt: "return" [`expression_list`] :keyword:`return` may only occur syntactically nested in a function definition, @@ -520,7 +525,7 @@ The :keyword:`!yield` statement single: function; generator exception: StopIteration -.. productionlist:: +.. productionlist:: python-grammar yield_stmt: `yield_expression` A :keyword:`yield` statement is semantically equivalent to a :ref:`yield @@ -555,7 +560,7 @@ The :keyword:`!raise` statement pair: raising; exception single: __traceback__ (exception attribute) -.. productionlist:: +.. productionlist:: python-grammar raise_stmt: "raise" [`expression` ["from" `expression`]] If no expressions are present, :keyword:`raise` re-raises the last exception @@ -658,7 +663,7 @@ The :keyword:`!break` statement statement: while pair: loop; statement -.. productionlist:: +.. productionlist:: python-grammar break_stmt: "break" :keyword:`break` may only occur syntactically nested in a :keyword:`for` or @@ -693,7 +698,7 @@ The :keyword:`!continue` statement pair: loop; statement keyword: finally -.. productionlist:: +.. productionlist:: python-grammar continue_stmt: "continue" :keyword:`continue` may only occur syntactically nested in a :keyword:`for` or @@ -720,7 +725,7 @@ The :keyword:`!import` statement exception: ImportError single: , (comma); import statement -.. productionlist:: +.. productionlist:: python-grammar import_stmt: "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])* : | "from" `relative_module` "import" `identifier` ["as" `identifier`] : ("," `identifier` ["as" `identifier`])* @@ -828,11 +833,13 @@ exists. Two dots means up one package level. Three dots is up two levels, etc. So if you execute ``from . import mod`` from a module in the ``pkg`` package then you will end up importing ``pkg.mod``. If you execute ``from ..subpkg2 import mod`` from within ``pkg.subpkg1`` you will import ``pkg.subpkg2.mod``. -The specification for relative imports is contained within :pep:`328`. +The specification for relative imports is contained in +the :ref:`relativeimports` section. :func:`importlib.import_module` is provided to support applications that determine dynamically the modules to be loaded. +.. audit-event:: import module,filename,sys.path,sys.meta_path,sys.path_hooks import .. _future: @@ -852,7 +859,7 @@ that introduce incompatible changes to the language. It allows use of the new features on a per-module basis before the release in which the feature becomes standard. -.. productionlist:: * +.. productionlist:: python-grammar future_stmt: "from" "__future__" "import" `feature` ["as" `identifier`] : ("," `feature` ["as" `identifier`])* : | "from" "__future__" "import" "(" `feature` ["as" `identifier`] @@ -930,7 +937,7 @@ The :keyword:`!global` statement triple: global; name; binding single: , (comma); identifier list -.. productionlist:: +.. productionlist:: python-grammar global_stmt: "global" `identifier` ("," `identifier`)* The :keyword:`global` statement is a declaration which holds for the entire @@ -975,7 +982,7 @@ The :keyword:`!nonlocal` statement .. index:: statement: nonlocal single: , (comma); identifier list -.. productionlist:: +.. productionlist:: python-grammar nonlocal_stmt: "nonlocal" `identifier` ("," `identifier`)* .. XXX add when implemented diff --git a/Doc/reference/toplevel_components.rst b/Doc/reference/toplevel_components.rst index d5ffb37b2e58cd..319c9de484241e 100644 --- a/Doc/reference/toplevel_components.rst +++ b/Doc/reference/toplevel_components.rst @@ -66,7 +66,7 @@ File input All input read from non-interactive files has the same form: -.. productionlist:: +.. productionlist:: python-grammar file_input: (NEWLINE | `statement`)* This syntax is used in the following situations: @@ -85,7 +85,7 @@ Interactive input Input in interactive mode is parsed using the following grammar: -.. productionlist:: +.. productionlist:: python-grammar interactive_input: [`stmt_list`] NEWLINE | `compound_stmt` NEWLINE Note that a (top-level) compound statement must be followed by a blank line in @@ -103,5 +103,5 @@ Expression input :func:`eval` is used for expression input. It ignores leading whitespace. The string argument to :func:`eval` must have the following form: -.. productionlist:: +.. productionlist:: python-grammar eval_input: `expression_list` NEWLINE* diff --git a/Doc/requirements.txt b/Doc/requirements.txt new file mode 100644 index 00000000000000..2b70af3a4fc6b9 --- /dev/null +++ b/Doc/requirements.txt @@ -0,0 +1,12 @@ +# Requirements to build the Python documentation + +# Sphinx version is pinned so that new versions that introduce new warnings +# won't suddenly cause build failures. Updating the version is fine as long +# as no warnings are raised by doing so. +sphinx==3.2.1 + +blurb + +# The theme used by the documentation is stored separately, so we need +# to install that as well. +python-docs-theme diff --git a/Doc/runtime.txt b/Doc/runtime.txt new file mode 100644 index 00000000000000..548d71365f0ec7 --- /dev/null +++ b/Doc/runtime.txt @@ -0,0 +1 @@ +3.7 \ No newline at end of file diff --git a/Doc/tools/extensions/asdl_highlight.py b/Doc/tools/extensions/asdl_highlight.py new file mode 100644 index 00000000000000..b1989e53957072 --- /dev/null +++ b/Doc/tools/extensions/asdl_highlight.py @@ -0,0 +1,54 @@ +import os +import sys +from pathlib import Path + +CPYTHON_ROOT = Path(__file__).resolve().parent.parent.parent.parent +sys.path.append(str(CPYTHON_ROOT / "Parser")) + +from pygments.lexer import RegexLexer, bygroups, include, words +from pygments.token import (Comment, Generic, Keyword, Name, Operator, + Punctuation, Text) + +from asdl import builtin_types +from sphinx.highlighting import lexers + +class ASDLLexer(RegexLexer): + name = "ASDL" + aliases = ["asdl"] + filenames = ["*.asdl"] + _name = r"([^\W\d]\w*)" + _text_ws = r"(\s*)" + + tokens = { + "ws": [ + (r"\n", Text), + (r"\s+", Text), + (r"--.*?$", Comment.Singleline), + ], + "root": [ + include("ws"), + ( + r"(module)" + _text_ws + _name, + bygroups(Keyword, Text, Name.Tag), + ), + ( + r"(\w+)(\*\s|\?\s|\s)(\w+)", + bygroups(Name.Builtin.Pseudo, Operator, Name), + ), + (words(builtin_types), Name.Builtin), + (r"attributes", Name.Builtin), + ( + _name + _text_ws + "(=)", + bygroups(Name, Text, Operator), + ), + (_name, Name.Class), + (r"\|", Operator), + (r"{|}|\(|\)", Punctuation), + (r".", Text), + ], + } + + +def setup(app): + lexers["asdl"] = ASDLLexer() + return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/peg_highlight.py b/Doc/tools/extensions/peg_highlight.py new file mode 100644 index 00000000000000..9a2acb7f320ba6 --- /dev/null +++ b/Doc/tools/extensions/peg_highlight.py @@ -0,0 +1,75 @@ +from pygments.lexer import RegexLexer, bygroups, include +from pygments.token import Comment, Generic, Keyword, Name, Operator, Punctuation, Text + +from sphinx.highlighting import lexers + + +class PEGLexer(RegexLexer): + """Pygments Lexer for PEG grammar (.gram) files + + This lexer strips the following elements from the grammar: + + - Meta-tags + - Variable assignments + - Actions + - Lookaheads + - Rule types + - Rule options + - Rules named `invalid_*` or `incorrect_*` + """ + + name = "PEG" + aliases = ["peg"] + filenames = ["*.gram"] + _name = r"([^\W\d]\w*)" + _text_ws = r"(\s*)" + + tokens = { + "ws": [(r"\n", Text), (r"\s+", Text), (r"#.*$", Comment.Singleline),], + "lookaheads": [ + (r"(?<=\|\s)(&\w+\s?)", bygroups(None)), + (r"(?<=\|\s)(&'.+'\s?)", bygroups(None)), + (r'(?<=\|\s)(&".+"\s?)', bygroups(None)), + (r"(?<=\|\s)(&\(.+\)\s?)", bygroups(None)), + ], + "metas": [ + (r"(@\w+ '''(.|\n)+?''')", bygroups(None)), + (r"^(@.*)$", bygroups(None)), + ], + "actions": [(r"{(.|\n)+?}", bygroups(None)),], + "strings": [ + (r"'\w+?'", Keyword), + (r'"\w+?"', Keyword), + (r"'\W+?'", Text), + (r'"\W+?"', Text), + ], + "variables": [(_name + _text_ws + r"(\[.*\])?" + _text_ws + "(=)", bygroups(None, None, None, None, None),),], + "invalids": [ + (r"^(\s+\|\s+invalid_\w+\s*\n)", bygroups(None)), + (r"^(\s+\|\s+incorrect_\w+\s*\n)", bygroups(None)), + (r"^(#.*invalid syntax.*(?:.|\n)*)", bygroups(None),), + ], + "root": [ + include("invalids"), + include("ws"), + include("lookaheads"), + include("metas"), + include("actions"), + include("strings"), + include("variables"), + (r"\b(?!(NULL|EXTRA))([A-Z_]+)\b\s*(?!\()", Text,), + ( + r"^\s*" + _name + r"\s*" + r"(\[.*\])?" + r"\s*" + r"(\(.+\))?" + r"\s*(:)", + bygroups(Name.Function, None, None, Punctuation), + ), + (_name, Name.Function), + (r"[\||\.|\+|\*|\?]", Operator), + (r"{|}|\(|\)|\[|\]", Punctuation), + (r".", Text), + ], + } + + +def setup(app): + lexers["peg"] = PEGLexer() + return {"version": "1.0", "parallel_read_safe": True} diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 66317430881b44..80fbd96d56fdc0 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -22,13 +22,21 @@ from sphinx import addnodes from sphinx.builders import Builder +try: + from sphinx.errors import NoUri +except ImportError: + from sphinx.environment import NoUri from sphinx.locale import translators -from sphinx.util import status_iterator +from sphinx.util import status_iterator, logging from sphinx.util.nodes import split_explicit_title -from sphinx.writers.html import HTMLTranslator from sphinx.writers.text import TextWriter, TextTranslator from sphinx.writers.latex import LaTeXTranslator -from sphinx.domains.python import PyModulelevel, PyClassmember + +try: + from sphinx.domains.python import PyFunction, PyMethod +except ImportError: + from sphinx.domains.python import PyClassmember as PyMethod + from sphinx.domains.python import PyModulelevel as PyFunction # Support for checking for suspicious markup @@ -45,37 +53,6 @@ Body.enum.converters['lowerroman'] = \ Body.enum.converters['upperroman'] = lambda x: None -# monkey-patch HTML and LaTeX translators to keep doctest blocks in the -# doctest docs themselves -orig_visit_literal_block = HTMLTranslator.visit_literal_block -orig_depart_literal_block = LaTeXTranslator.depart_literal_block - - -def new_visit_literal_block(self, node): - meta = self.builder.env.metadata[self.builder.current_docname] - old_trim_doctest_flags = self.highlighter.trim_doctest_flags - if 'keepdoctest' in meta: - self.highlighter.trim_doctest_flags = False - try: - orig_visit_literal_block(self, node) - finally: - self.highlighter.trim_doctest_flags = old_trim_doctest_flags - - -def new_depart_literal_block(self, node): - meta = self.builder.env.metadata[self.curfilestack[-1]] - old_trim_doctest_flags = self.highlighter.trim_doctest_flags - if 'keepdoctest' in meta: - self.highlighter.trim_doctest_flags = False - try: - orig_depart_literal_block(self, node) - finally: - self.highlighter.trim_doctest_flags = old_trim_doctest_flags - - -HTMLTranslator.visit_literal_block = new_visit_literal_block -LaTeXTranslator.depart_literal_block = new_depart_literal_block - # Support for marking up and linking to bugs.python.org issues @@ -151,6 +128,142 @@ def run(self): return [pnode] +# Support for documenting audit event + +def audit_events_purge(app, env, docname): + """This is to remove from env.all_audit_events old traces of removed + documents. + """ + if not hasattr(env, 'all_audit_events'): + return + fresh_all_audit_events = {} + for name, event in env.all_audit_events.items(): + event["source"] = [(d, t) for d, t in event["source"] if d != docname] + if event["source"]: + # Only keep audit_events that have at least one source. + fresh_all_audit_events[name] = event + env.all_audit_events = fresh_all_audit_events + + +def audit_events_merge(app, env, docnames, other): + """In Sphinx parallel builds, this merges env.all_audit_events from + subprocesses. + + all_audit_events is a dict of names, with values like: + {'source': [(docname, target), ...], 'args': args} + """ + if not hasattr(other, 'all_audit_events'): + return + if not hasattr(env, 'all_audit_events'): + env.all_audit_events = {} + for name, value in other.all_audit_events.items(): + if name in env.all_audit_events: + env.all_audit_events[name]["source"].extend(value["source"]) + else: + env.all_audit_events[name] = value + + +class AuditEvent(Directive): + + has_content = True + required_arguments = 1 + optional_arguments = 2 + final_argument_whitespace = True + + _label = [ + "Raises an :ref:`auditing event ` {name} with no arguments.", + "Raises an :ref:`auditing event ` {name} with argument {args}.", + "Raises an :ref:`auditing event ` {name} with arguments {args}.", + ] + + @property + def logger(self): + cls = type(self) + return logging.getLogger(cls.__module__ + "." + cls.__name__) + + def run(self): + name = self.arguments[0] + if len(self.arguments) >= 2 and self.arguments[1]: + args = (a.strip() for a in self.arguments[1].strip("'\"").split(",")) + args = [a for a in args if a] + else: + args = [] + + label = translators['sphinx'].gettext(self._label[min(2, len(args))]) + text = label.format(name="``{}``".format(name), + args=", ".join("``{}``".format(a) for a in args if a)) + + env = self.state.document.settings.env + if not hasattr(env, 'all_audit_events'): + env.all_audit_events = {} + + new_info = { + 'source': [], + 'args': args + } + info = env.all_audit_events.setdefault(name, new_info) + if info is not new_info: + if not self._do_args_match(info['args'], new_info['args']): + self.logger.warn( + "Mismatched arguments for audit-event {}: {!r} != {!r}" + .format(name, info['args'], new_info['args']) + ) + + ids = [] + try: + target = self.arguments[2].strip("\"'") + except (IndexError, TypeError): + target = None + if not target: + target = "audit_event_{}_{}".format( + re.sub(r'\W', '_', name), + len(info['source']), + ) + ids.append(target) + + info['source'].append((env.docname, target)) + + pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) + if self.content: + self.state.nested_parse(self.content, self.content_offset, pnode) + else: + n, m = self.state.inline_text(text, self.lineno) + pnode.extend(n + m) + + return [pnode] + + # This list of sets are allowable synonyms for event argument names. + # If two names are in the same set, they are treated as equal for the + # purposes of warning. This won't help if number of arguments is + # different! + _SYNONYMS = [ + {"file", "path", "fd"}, + ] + + def _do_args_match(self, args1, args2): + if args1 == args2: + return True + if len(args1) != len(args2): + return False + for a1, a2 in zip(args1, args2): + if a1 == a2: + continue + if any(a1 in s and a2 in s for s in self._SYNONYMS): + continue + return False + return True + + +class audit_event_list(nodes.General, nodes.Element): + pass + + +class AuditEventListDirective(Directive): + + def run(self): + return [audit_event_list('')] + + # Support for documenting decorators class PyDecoratorMixin(object): @@ -163,17 +276,18 @@ def needs_arglist(self): return False -class PyDecoratorFunction(PyDecoratorMixin, PyModulelevel): +class PyDecoratorFunction(PyDecoratorMixin, PyFunction): def run(self): # a decorator function is a function after all self.name = 'py:function' - return PyModulelevel.run(self) + return PyFunction.run(self) -class PyDecoratorMethod(PyDecoratorMixin, PyClassmember): +# TODO: Use sphinx.domains.python.PyDecoratorMethod when possible +class PyDecoratorMethod(PyDecoratorMixin, PyMethod): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) class PyCoroutineMixin(object): @@ -190,31 +304,31 @@ def handle_signature(self, sig, signode): return ret -class PyCoroutineFunction(PyCoroutineMixin, PyModulelevel): +class PyCoroutineFunction(PyCoroutineMixin, PyFunction): def run(self): self.name = 'py:function' - return PyModulelevel.run(self) + return PyFunction.run(self) -class PyCoroutineMethod(PyCoroutineMixin, PyClassmember): +class PyCoroutineMethod(PyCoroutineMixin, PyMethod): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) -class PyAwaitableFunction(PyAwaitableMixin, PyClassmember): +class PyAwaitableFunction(PyAwaitableMixin, PyFunction): def run(self): self.name = 'py:function' - return PyClassmember.run(self) + return PyFunction.run(self) -class PyAwaitableMethod(PyAwaitableMixin, PyClassmember): +class PyAwaitableMethod(PyAwaitableMixin, PyMethod): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) -class PyAbstractMethod(PyClassmember): +class PyAbstractMethod(PyMethod): def handle_signature(self, sig, signode): ret = super(PyAbstractMethod, self).handle_signature(sig, signode) @@ -224,7 +338,7 @@ def handle_signature(self, sig, signode): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) # Support for documenting version of removal in deprecations @@ -236,7 +350,8 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _label = 'Deprecated since version {deprecated}, will be removed in version {removed}' + _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}' + _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}' def run(self): node = addnodes.versionmodified() @@ -244,7 +359,15 @@ def run(self): node['type'] = 'deprecated-removed' version = (self.arguments[0], self.arguments[1]) node['version'] = version - label = translators['sphinx'].gettext(self._label) + env = self.state.document.settings.env + current_version = tuple(int(e) for e in env.config.version.split('.')) + removed_version = tuple(int(e) for e in self.arguments[1].split('.')) + if current_version < removed_version: + label = self._deprecated_label + else: + label = self._removed_label + + label = translators['sphinx'].gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], @@ -271,7 +394,7 @@ def run(self): translatable=False) node.append(para) env = self.state.document.settings.env - env.note_versionchange('deprecated', version[0], node, self.lineno) + env.get_domain('changeset').note_changeset(node) return [node] + messages @@ -355,7 +478,7 @@ def write(self, *ignored): 'building topics... ', length=len(pydoc_topic_labels)): if label not in self.env.domaindata['std']['labels']: - self.warn('label %r not in documentation' % label) + self.env.logger.warn('label %r not in documentation' % label) continue docname, labelid, sectname = self.env.domaindata['std']['labels'][label] doctree = self.env.get_and_resolve_doctree(docname, self) @@ -419,11 +542,77 @@ def parse_pdb_command(env, sig, signode): return fullname +def process_audit_events(app, doctree, fromdocname): + for node in doctree.traverse(audit_event_list): + break + else: + return + + env = app.builder.env + + table = nodes.table(cols=3) + group = nodes.tgroup( + '', + nodes.colspec(colwidth=30), + nodes.colspec(colwidth=55), + nodes.colspec(colwidth=15), + cols=3, + ) + head = nodes.thead() + body = nodes.tbody() + + table += group + group += head + group += body + + row = nodes.row() + row += nodes.entry('', nodes.paragraph('', nodes.Text('Audit event'))) + row += nodes.entry('', nodes.paragraph('', nodes.Text('Arguments'))) + row += nodes.entry('', nodes.paragraph('', nodes.Text('References'))) + head += row + + for name in sorted(getattr(env, "all_audit_events", ())): + audit_event = env.all_audit_events[name] + + row = nodes.row() + node = nodes.paragraph('', nodes.Text(name)) + row += nodes.entry('', node) + + node = nodes.paragraph() + for i, a in enumerate(audit_event['args']): + if i: + node += nodes.Text(", ") + node += nodes.literal(a, nodes.Text(a)) + row += nodes.entry('', node) + + node = nodes.paragraph() + backlinks = enumerate(sorted(set(audit_event['source'])), start=1) + for i, (doc, label) in backlinks: + if isinstance(label, str): + ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True) + try: + ref['refuri'] = "{}#{}".format( + app.builder.get_relative_uri(fromdocname, doc), + label, + ) + except NoUri: + continue + node += ref + row += nodes.entry('', node) + + body += row + + for node in doctree.traverse(audit_event_list): + node.replace_self(table) + + def setup(app): app.add_role('issue', issue_role) app.add_role('source', source_role) app.add_directive('impl-detail', ImplementationDetail) app.add_directive('availability', Availability) + app.add_directive('audit-event', AuditEvent) + app.add_directive('audit-event-table', AuditEventListDirective) app.add_directive('deprecated-removed', DeprecatedRemoved) app.add_builder(PydocTopicsBuilder) app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) @@ -438,4 +627,7 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) + app.connect('doctree-resolved', process_audit_events) + app.connect('env-merge-info', audit_events_merge) + app.connect('env-purge-doc', audit_events_purge) return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/suspicious.py b/Doc/tools/extensions/suspicious.py index 494efabc46234e..9e814fb94d2b56 100644 --- a/Doc/tools/extensions/suspicious.py +++ b/Doc/tools/extensions/suspicious.py @@ -115,10 +115,12 @@ def write_doc(self, docname, doctree): def finish(self): unused_rules = [rule for rule in self.rules if not rule.used] if unused_rules: - self.warn('Found %s/%s unused rules:' % - (len(unused_rules), len(self.rules))) - for rule in unused_rules: - self.logger.info(repr(rule)) + self.logger.warning( + 'Found %s/%s unused rules: %s' % ( + len(unused_rules), len(self.rules), + ''.join(repr(rule) for rule in unused_rules), + ) + ) return def check_issue(self, line, lineno, issue): @@ -151,14 +153,15 @@ def report_issue(self, text, lineno, issue): self.any_issue = True self.write_log_entry(lineno, issue, text) if py3: - self.warn('[%s:%d] "%s" found in "%-.120s"' % - (self.docname, lineno, issue, text)) + self.logger.warning('[%s:%d] "%s" found in "%-.120s"' % + (self.docname, lineno, issue, text)) else: - self.warn('[%s:%d] "%s" found in "%-.120s"' % ( - self.docname.encode(sys.getdefaultencoding(),'replace'), - lineno, - issue.encode(sys.getdefaultencoding(),'replace'), - text.strip().encode(sys.getdefaultencoding(),'replace'))) + self.logger.warning( + '[%s:%d] "%s" found in "%-.120s"' % ( + self.docname.encode(sys.getdefaultencoding(),'replace'), + lineno, + issue.encode(sys.getdefaultencoding(),'replace'), + text.strip().encode(sys.getdefaultencoding(),'replace'))) self.app.statuscode = 1 def write_log_entry(self, lineno, issue, text): diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index 346b31494e60f9..c51f178ce645c0 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,7 +10,9 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.8': 'dev (3.8)', + '3.10': 'dev (3.10)', + '3.9': 'pre (3.9)', + '3.8': '3.8', '3.7': '3.7', '3.6': '3.6', '3.5': '3.5', @@ -22,6 +24,7 @@ 'fr': 'French', 'ja': 'Japanese', 'ko': 'Korean', + 'pt-br': 'Brazilian Portuguese', 'zh-cn': 'Simplified Chinese', }; diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 3672955bf55b11..7be8d0abd69a57 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -2,6 +2,7 @@ c-api/arg,,:ref,"PyArg_ParseTuple(args, ""O|O:ref"", &object, &callback)" c-api/list,,:high,list[low:high] c-api/sequence,,:i2,del o[i1:i2] c-api/sequence,,:i2,o[i1:i2] +c-api/tuple,,:high,p[low:high] c-api/unicode,,:end,str[start:end] c-api/unicode,,:start,unicode[start:start+length] distutils/examples,267,`,This is the description of the ``foobar`` package. @@ -81,6 +82,7 @@ howto/ipaddress,,::,IPv6Address('2001:db8::ffff:ffff') howto/ipaddress,,:ffff,IPv6Address('2001:db8::ffff:ffff') howto/logging,,:And,"WARNING:And this, too" howto/logging,,:And,"WARNING:root:And this, too" +howto/logging,,:And,"ERROR:root:And non-ASCII stuff, too, like " howto/logging,,:Doing,INFO:root:Doing something howto/logging,,:Finished,INFO:root:Finished howto/logging,,:logger,severity:logger name:message @@ -90,6 +92,7 @@ howto/logging,,:root,DEBUG:root:This message should go to the log file howto/logging,,:root,INFO:root:Doing something howto/logging,,:root,INFO:root:Finished howto/logging,,:root,INFO:root:So should this +howto/logging,,:root,"ERROR:root:And non-ASCII stuff, too, like " howto/logging,,:root,INFO:root:Started howto/logging,,:root,"WARNING:root:And this, too" howto/logging,,:root,WARNING:root:Look before you leap! @@ -105,6 +108,8 @@ howto/pyporting,,::,Programming Language :: Python :: 3 howto/regex,,::, howto/regex,,:foo,(?:foo) howto/urllib2,,:password,"""joe:password@example.com""" +library/ast,,:upper,lower:upper +library/ast,,:step,lower:upper:step library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)]," library/bisect,32,:hi,all(val >= x for val in a[i:hi]) library/bisect,42,:hi,all(val > x for val in a[i:hi]) @@ -144,6 +149,8 @@ library/ipaddress,,:db8,>>> ipaddress.IPv6Address('2001:db8::1000') library/ipaddress,,::,>>> ipaddress.IPv6Address('2001:db8::1000') library/ipaddress,,:db8,IPv6Address('2001:db8::1000') library/ipaddress,,::,IPv6Address('2001:db8::1000') +library/ipaddress,,::,IPv6Address('ff02::5678%1') +library/ipaddress,,::,fe80::1234 library/ipaddress,,:db8,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer" library/ipaddress,,::,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer" library/ipaddress,,::,"""::abc:7:def""" @@ -198,7 +205,7 @@ library/readline,,:bind,"python:bind ^I rl_complete" library/smtplib,,:port,method must support that as well as a regular host:port library/socket,,::,'5aef:2b::8' library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])" -library/socket,,:len,fds.fromstring(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) +library/socket,,:len,fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) library/sqlite3,,:age,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" library/sqlite3,,:memory, library/sqlite3,,:who,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" @@ -216,8 +223,6 @@ library/stdtypes,,:len,s[len(s):len(s)] library/stdtypes,,::,>>> y = m[::2] library/stdtypes,,::,>>> z = y[::-2] library/string,,`,"!""#$%&'()*+,-./:;<=>?@[\]^_`{|}~" -library/subprocess,,`,"output=`dmesg | grep hda`" -library/subprocess,,`,"output=`mycmd myarg`" library/tarfile,,:bz2, library/tarfile,,:compression,filemode[:compression] library/tarfile,,:gz, @@ -236,10 +241,12 @@ library/urllib.request,,:close,Connection:close library/urllib.request,,:port,:port library/urllib.request,,:lang,"xmlns=""http://www.w3.org/1999/xhtml"" xml:lang=""en"" lang=""en"">\n\n\n" library/urllib.request,,:password,"""joe:password@python.org""" +library/urllib.parse,,:scheme, +library/urllib.parse,,:scheme,URL:scheme://host/path library/uuid,,:uuid,urn:uuid:12345678-1234-5678-1234-567812345678 -library/venv,,:param,":param nodist: If True, setuptools and pip are not installed into the" +library/venv,,:param,":param nodist: If true, setuptools and pip are not installed into the" library/venv,,:param,":param progress: If setuptools or pip are installed, the progress of the" -library/venv,,:param,":param nopip: If True, pip is not installed into the created" +library/venv,,:param,":param nopip: If true, pip is not installed into the created" library/venv,,:param,:param context: The information for the virtual environment library/xmlrpc.client,,:nil,ex:nil library/xmlrpc.client,,:pass,http://user:pass@host:port/path @@ -331,11 +338,13 @@ library/xml.etree.elementtree,,:character,Commander Clement library/xml.etree.elementtree,,:actor,"for actor in root.findall('real_person:actor', ns):" library/xml.etree.elementtree,,:name,"name = actor.find('real_person:name', ns)" library/xml.etree.elementtree,,:character,"for char in actor.findall('role:character', ns):" +library/xml.etree.elementtree,,:xi, +library/xml.etree.elementtree,,:include, +library/xml.etree.elementtree,,:include, Copyright (c) . library/zipapp,,:main,"$ python -m zipapp myapp -m ""myapp:main""" library/zipapp,,:fn,"pkg.mod:fn" library/zipapp,,:callable,"pkg.module:callable" library/stdtypes,,::,>>> m[::2].tolist() -library/sys,,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine: whatsnew/3.5,,:root,'WARNING:root:warning\n' whatsnew/3.5,,:warning,'WARNING:root:warning\n' whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') @@ -350,3 +359,6 @@ whatsnew/3.7,,::,error::BytesWarning whatsnew/changelog,,::,error::BytesWarning whatsnew/changelog,,::,default::BytesWarning whatsnew/changelog,,::,default::DeprecationWarning +library/importlib.metadata,,:main,"EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts')" +library/importlib.metadata,,`,loading the metadata for packages for the indicated ``context``. +library/re,,`,"`" diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 1a99b18bbb268e..987c63a7f3575c 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -12,8 +12,7 @@

Download Python {{ release }} Documentation

{% if last_updated %}

Last updated on: {{ last_updated }}.

{% endif %}

To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table. The numbers -in the table are the size of the download files in megabytes.

+Python in one of various formats, follow one of links in this table.

@@ -41,7 +40,7 @@

Download Python {{ release }} Documentation

These archives contain all the content in the documentation.

-

HTML Help (.chm) files are made available in the "Windows" section +

HTML Help (.chm) files are made available in the "Windows" section on the Python download page.

diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 8d94137b01b519..68ae3ad148ec27 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -5,3 +5,4 @@ {% trans %}CPython implementation detail:{% endtrans %} {% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} +{% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 3666af92f0da47..7a40be77aa1295 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,11 +2,13 @@

{% trans %}Download{% endtrans %}

{% trans %}Download these documents{% endtrans %}

{% trans %}Docs by version{% endtrans %}

diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index c3992245614082..17592d74a4eb52 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -1,5 +1,15 @@ {% extends "!layout.html" %} +{% block header %} +{%- if outdated %} +
+ {% trans %}This document is for an old version of Python that is no longer supported. + You should upgrade, and read the {% endtrans %} + {% trans %} Python documentation for the current stable release{% endtrans %}. +
+{%- endif %} +{% endblock %} + {% block rootrellink %} {{ super() }}
  • diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 7619ccbc1f36e9..0d780e3ba89643 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -114,8 +114,8 @@ accessible. "Directly accessible" here means that an unqualified reference to a name attempts to find the name in the namespace. Although scopes are determined statically, they are used dynamically. At any -time during execution, there are at least three nested scopes whose namespaces -are directly accessible: +time during execution, there are 3 or 4 nested scopes whose namespaces are +directly accessible: * the innermost scope, which is searched first, contains the local names * the scopes of any enclosing functions, which are searched starting with the @@ -143,10 +143,10 @@ language definition is evolving towards static name resolution, at "compile" time, so don't rely on dynamic name resolution! (In fact, local variables are already determined statically.) -A special quirk of Python is that -- if no :keyword:`global` statement is in -effect -- assignments to names always go into the innermost scope. Assignments -do not copy data --- they just bind names to objects. The same is true for -deletions: the statement ``del x`` removes the binding of ``x`` from the +A special quirk of Python is that -- if no :keyword:`global` or :keyword:`nonlocal` +statement is in effect -- assignments to names always go into the innermost scope. +Assignments do not copy data --- they just bind names to objects. The same is true +for deletions: the statement ``del x`` removes the binding of ``x`` from the namespace referenced by the local scope. In fact, all operations that introduce new names use the local scope: in particular, :keyword:`import` statements and function definitions bind the module or function name in the local scope. @@ -323,7 +323,7 @@ Instance Objects Now what can we do with instance objects? The only operations understood by instance objects are attribute references. There are two kinds of valid -attribute names, data attributes and methods. +attribute names: data attributes and methods. *data attributes* correspond to "instance variables" in Smalltalk, and to "data members" in C++. Data attributes need not be declared; like local variables, @@ -475,12 +475,20 @@ Random Remarks .. These should perhaps be placed more carefully... -Data attributes override method attributes with the same name; to avoid -accidental name conflicts, which may cause hard-to-find bugs in large programs, -it is wise to use some kind of convention that minimizes the chance of -conflicts. Possible conventions include capitalizing method names, prefixing -data attribute names with a small unique string (perhaps just an underscore), or -using verbs for methods and nouns for data attributes. +If the same attribute name occurs in both an instance and in a class, +then attribute lookup prioritizes the instance:: + + >>> class Warehouse: + purpose = 'storage' + region = 'west' + + >>> w1 = Warehouse() + >>> print(w1.purpose, w1.region) + storage west + >>> w2 = Warehouse() + >>> w2.region = 'east' + >>> print(w2.purpose, w2.region) + storage east Data attributes may be referenced by methods as well as by ordinary users ("clients") of an object. In other words, classes are not usable to implement @@ -841,7 +849,7 @@ defines :meth:`__next__`, then :meth:`__iter__` can just return ``self``:: Generators ========== -:term:`Generator`\s are a simple and powerful tool for creating iterators. They +:term:`Generators ` are a simple and powerful tool for creating iterators. They are written like regular functions but use the :keyword:`yield` statement whenever they want to return data. Each time :func:`next` is called on it, the generator resumes where it left off (it remembers all the data values and which diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 905734539c6851..b8aec2b04f13fb 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -4,8 +4,8 @@ More Control Flow Tools *********************** -Besides the :keyword:`while` statement just introduced, Python knows the usual -control flow statements known from other languages, with some twists. +Besides the :keyword:`while` statement just introduced, Python uses the usual +flow control statements known from other languages, with some twists. .. _tut-if: @@ -66,20 +66,23 @@ they appear in the sequence. For example (no pun intended): window 6 defenestrate 12 -If you need to modify the sequence you are iterating over while inside the loop -(for example to duplicate selected items), it is recommended that you first -make a copy. Iterating over a sequence does not implicitly make a copy. The -slice notation makes this especially convenient:: +Code that modifies a collection while iterating over that same collection can +be tricky to get right. Instead, it is usually more straight-forward to loop +over a copy of the collection or to create a new collection:: - >>> for w in words[:]: # Loop over a slice copy of the entire list. - ... if len(w) > 6: - ... words.insert(0, w) - ... - >>> words - ['defenestrate', 'cat', 'window', 'defenestrate'] + # Create a sample collection + users = {'Hans': 'active', 'Éléonore': 'inactive', '景太郎': 'active'} + + # Strategy: Iterate over a copy + for user, status in users.copy().items(): + if status == 'inactive': + del users[user] -With ``for w in words:``, the example would attempt to create an infinite list, -inserting ``defenestrate`` over and over again. + # Strategy: Create a new collection + active_users = {} + for user, status in users.items(): + if status == 'active': + active_users[user] = status .. _tut-range: @@ -139,18 +142,24 @@ but in fact it isn't. It is an object which returns the successive items of the desired sequence when you iterate over it, but it doesn't really make the list, thus saving space. -We say such an object is *iterable*, that is, suitable as a target for +We say such an object is :term:`iterable`, that is, suitable as a target for functions and constructs that expect something from which they can -obtain successive items until the supply is exhausted. We have seen that -the :keyword:`for` statement is such an *iterator*. The function :func:`list` -is another; it creates lists from iterables:: +obtain successive items until the supply is exhausted. We have seen that +the :keyword:`for` statement is such a construct, while an example of a function +that takes an iterable is :func:`sum`:: + >>> sum(range(4)) # 0 + 1 + 2 + 3 + 6 - >>> list(range(5)) - [0, 1, 2, 3, 4] +Later we will see more functions that return iterables and take iterables as +arguments. Lastly, maybe you are curious about how to get a list from a range. +Here is the solution:: -Later we will see more functions that return iterables and take iterables as argument. + >>> list(range(4)) + [0, 1, 2, 3] +In chapter :ref:`tut-structures`, we will discuss in more detail about +:func:`list`. .. _tut-break: @@ -161,7 +170,7 @@ The :keyword:`break` statement, like in C, breaks out of the innermost enclosing :keyword:`for` or :keyword:`while` loop. Loop statements may have an :keyword:`!else` clause; it is executed when the loop -terminates through exhaustion of the list (with :keyword:`for`) or when the +terminates through exhaustion of the iterable (with :keyword:`for`) or when the condition becomes false (with :keyword:`while`), but not when the loop is terminated by a :keyword:`break` statement. This is exemplified by the following loop, which searches for prime numbers:: @@ -188,8 +197,8 @@ following loop, which searches for prime numbers:: the :keyword:`for` loop, **not** the :keyword:`if` statement.) When used with a loop, the ``else`` clause has more in common with the -``else`` clause of a :keyword:`try` statement than it does that of -:keyword:`if` statements: a :keyword:`!try` statement's ``else`` clause runs +``else`` clause of a :keyword:`try` statement than it does with that of +:keyword:`if` statements: a :keyword:`try` statement's ``else`` clause runs when no exception occurs, and a loop's ``else`` clause runs when no ``break`` occurs. For more on the :keyword:`!try` statement and exceptions, see :ref:`tut-handling`. @@ -201,15 +210,15 @@ iteration of the loop:: ... if num % 2 == 0: ... print("Found an even number", num) ... continue - ... print("Found a number", num) + ... print("Found an odd number", num) Found an even number 2 - Found a number 3 + Found an odd number 3 Found an even number 4 - Found a number 5 + Found an odd number 5 Found an even number 6 - Found a number 7 + Found an odd number 7 Found an even number 8 - Found a number 9 + Found an odd number 9 .. _tut-pass: @@ -279,9 +288,11 @@ variables of the function. More precisely, all variable assignments in a function store the value in the local symbol table; whereas variable references first look in the local symbol table, then in the local symbol tables of enclosing functions, then in the global symbol table, and finally in the table -of built-in names. Thus, global variables cannot be directly assigned a value -within a function (unless named in a :keyword:`global` statement), although they -may be referenced. +of built-in names. Thus, global variables and variables of enclosing functions +cannot be directly assigned a value within a function (unless, for global +variables, named in a :keyword:`global` statement, or, for variables of enclosing +functions, named in a :keyword:`nonlocal` statement), although they may be +referenced. The actual parameters (arguments) to a function call are introduced in the local symbol table of the called function when it is called; thus, arguments are @@ -289,11 +300,10 @@ passed using *call by value* (where the *value* is always an object *reference*, not the value of the object). [#]_ When a function calls another function, a new local symbol table is created for that call. -A function definition introduces the function name in the current symbol table. -The value of the function name has a type that is recognized by the interpreter -as a user-defined function. This value can be assigned to another name which -can then also be used as a function. This serves as a general renaming -mechanism:: +A function definition associates the function name with the function object in +the current symbol table. The interpreter recognizes the object pointed to by +that name as a user-defined function. Other names can also point to that same +function object and can also be used to access the function:: >>> fib @@ -482,9 +492,9 @@ When a final formal parameter of the form ``**name`` is present, it receives a dictionary (see :ref:`typesmapping`) containing all keyword arguments except for those corresponding to a formal parameter. This may be combined with a formal parameter of the form ``*name`` (described in the next subsection) which -receives a tuple containing the positional arguments beyond the formal parameter -list. (``*name`` must occur before ``**name``.) For example, if we define a -function like this:: +receives a :ref:`tuple ` containing the positional +arguments beyond the formal parameter list. (``*name`` must occur +before ``**name``.) For example, if we define a function like this:: def cheeseshop(kind, *arguments, **keywords): print("-- Do you have any", kind, "?") @@ -519,6 +529,176 @@ and of course it would print: Note that the order in which the keyword arguments are printed is guaranteed to match the order in which they were provided in the function call. +Special parameters +------------------ + +By default, arguments may be passed to a Python function either by position +or explicitly by keyword. For readability and performance, it makes sense to +restrict the way arguments can be passed so that a developer need only look +at the function definition to determine if items are passed by position, by +position or keyword, or by keyword. + +A function definition may look like: + +.. code-block:: none + + def f(pos1, pos2, /, pos_or_kwd, *, kwd1, kwd2): + ----------- ---------- ---------- + | | | + | Positional or keyword | + | - Keyword only + -- Positional only + +where ``/`` and ``*`` are optional. If used, these symbols indicate the kind of +parameter by how the arguments may be passed to the function: +positional-only, positional-or-keyword, and keyword-only. Keyword parameters +are also referred to as named parameters. + +------------------------------- +Positional-or-Keyword Arguments +------------------------------- + +If ``/`` and ``*`` are not present in the function definition, arguments may +be passed to a function by position or by keyword. + +-------------------------- +Positional-Only Parameters +-------------------------- + +Looking at this in a bit more detail, it is possible to mark certain parameters +as *positional-only*. If *positional-only*, the parameters' order matters, and +the parameters cannot be passed by keyword. Positional-only parameters are +placed before a ``/`` (forward-slash). The ``/`` is used to logically +separate the positional-only parameters from the rest of the parameters. +If there is no ``/`` in the function definition, there are no positional-only +parameters. + +Parameters following the ``/`` may be *positional-or-keyword* or *keyword-only*. + +---------------------- +Keyword-Only Arguments +---------------------- + +To mark parameters as *keyword-only*, indicating the parameters must be passed +by keyword argument, place an ``*`` in the arguments list just before the first +*keyword-only* parameter. + +----------------- +Function Examples +----------------- + +Consider the following example function definitions paying close attention to the +markers ``/`` and ``*``:: + + >>> def standard_arg(arg): + ... print(arg) + ... + >>> def pos_only_arg(arg, /): + ... print(arg) + ... + >>> def kwd_only_arg(*, arg): + ... print(arg) + ... + >>> def combined_example(pos_only, /, standard, *, kwd_only): + ... print(pos_only, standard, kwd_only) + + +The first function definition, ``standard_arg``, the most familiar form, +places no restrictions on the calling convention and arguments may be +passed by position or keyword:: + + >>> standard_arg(2) + 2 + + >>> standard_arg(arg=2) + 2 + +The second function ``pos_only_arg`` is restricted to only use positional +parameters as there is a ``/`` in the function definition:: + + >>> pos_only_arg(1) + 1 + + >>> pos_only_arg(arg=1) + Traceback (most recent call last): + File "", line 1, in + TypeError: pos_only_arg() got an unexpected keyword argument 'arg' + +The third function ``kwd_only_args`` only allows keyword arguments as indicated +by a ``*`` in the function definition:: + + >>> kwd_only_arg(3) + Traceback (most recent call last): + File "", line 1, in + TypeError: kwd_only_arg() takes 0 positional arguments but 1 was given + + >>> kwd_only_arg(arg=3) + 3 + +And the last uses all three calling conventions in the same function +definition:: + + >>> combined_example(1, 2, 3) + Traceback (most recent call last): + File "", line 1, in + TypeError: combined_example() takes 2 positional arguments but 3 were given + + >>> combined_example(1, 2, kwd_only=3) + 1 2 3 + + >>> combined_example(1, standard=2, kwd_only=3) + 1 2 3 + + >>> combined_example(pos_only=1, standard=2, kwd_only=3) + Traceback (most recent call last): + File "", line 1, in + TypeError: combined_example() got an unexpected keyword argument 'pos_only' + + +Finally, consider this function definition which has a potential collision between the positional argument ``name`` and ``**kwds`` which has ``name`` as a key:: + + def foo(name, **kwds): + return 'name' in kwds + +There is no possible call that will make it return ``True`` as the keyword ``'name'`` +will always bind to the first parameter. For example:: + + >>> foo(1, **{'name': 2}) + Traceback (most recent call last): + File "", line 1, in + TypeError: foo() got multiple values for argument 'name' + >>> + +But using ``/`` (positional only arguments), it is possible since it allows ``name`` as a positional argument and ``'name'`` as a key in the keyword arguments:: + + def foo(name, /, **kwds): + return 'name' in kwds + >>> foo(1, **{'name': 2}) + True + +In other words, the names of positional-only parameters can be used in +``**kwds`` without ambiguity. + +----- +Recap +----- + +The use case will determine which parameters to use in the function definition:: + + def f(pos1, pos2, /, pos_or_kwd, *, kwd1, kwd2): + +As guidance: + +* Use positional-only if you want the name of the parameters to not be + available to the user. This is useful when parameter names have no real + meaning, if you want to enforce the order of the arguments when the function + is called or if you need to take some positional parameters and arbitrary + keywords. +* Use keyword-only when names have meaning and the function definition is + more understandable by being explicit with names or you want to prevent + users relying on the position of the argument being passed. +* For an API, use positional-only to prevent breaking API changes + if the parameter's name is modified in the future. .. _tut-arbitraryargs: @@ -742,7 +922,7 @@ extracted for you: bracketing constructs: ``a = f(1, 2) + g(3, 4)``. * Name your classes and functions consistently; the convention is to use - ``CamelCase`` for classes and ``lower_case_with_underscores`` for functions + ``UpperCamelCase`` for classes and ``lowercase_with_underscores`` for functions and methods. Always use ``self`` as the name for the first method argument (see :ref:`tut-firstclasses` for more on classes and methods). diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 01e437bb5da858..ff4c797f66cd63 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -125,6 +125,13 @@ only modify the list have no return value printed -- they return the default ``None``. [1]_ This is a design principle for all mutable data structures in Python. +Another thing you might notice is that not all data can be sorted or +compared. For instance, ``[None, 'hello', 10]`` doesn't sort because +integers can't be compared to strings and *None* can't be compared to +other types. Also, there are some types that don't have a defined +ordering relation. For example, ``3+4j < 5+7j`` isn't a valid +comparison. + .. _tut-lists-as-stacks: @@ -606,6 +613,21 @@ direction and then call the :func:`reversed` function. :: To loop over a sequence in sorted order, use the :func:`sorted` function which returns a new sorted list while leaving the source unaltered. :: + >>> basket = ['apple', 'orange', 'apple', 'pear', 'orange', 'banana'] + >>> for i in sorted(basket): + ... print(i) + ... + apple + apple + banana + orange + orange + pear + +Using :func:`set` on a sequence eliminates duplicate elements. The use of +:func:`sorted` in combination with :func:`set` over a sequence is an idiomatic +way to loop over unique elements of the sequence in sorted order. :: + >>> basket = ['apple', 'orange', 'apple', 'pear', 'orange', 'banana'] >>> for f in sorted(set(basket)): ... print(f) @@ -668,10 +690,11 @@ to a variable. For example, :: >>> non_null 'Trondheim' -Note that in Python, unlike C, assignment cannot occur inside expressions. C -programmers may grumble about this, but it avoids a common class of problems -encountered in C programs: typing ``=`` in an expression when ``==`` was -intended. +Note that in Python, unlike C, assignment inside expressions must be done +explicitly with the +:ref:`walrus operator ` ``:=``. +This avoids a common class of problems encountered in C programs: typing ``=`` +in an expression when ``==`` was intended. .. _tut-comparing: diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 4e287bbd8d29ff..0ce96466e8c286 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -67,7 +67,7 @@ The rest of the line provides detail based on the type of exception and what caused it. The preceding part of the error message shows the context where the exception -happened, in the form of a stack traceback. In general it contains a stack +occurred, in the form of a stack traceback. In general it contains a stack traceback listing source lines; however, it will not display lines read from standard input. @@ -267,6 +267,53 @@ re-raise the exception:: NameError: HiThere +.. _tut-exception-chaining: + +Exception Chaining +================== + +The :keyword:`raise` statement allows an optional :keyword:`from` which enables +chaining exceptions by setting the ``__cause__`` attribute of the raised +exception. For example:: + + raise RuntimeError from OSError + +This can be useful when you are transforming exceptions. For example:: + + >>> def func(): + ... raise IOError + ... + >>> try: + ... func() + ... except IOError as exc: + ... raise RuntimeError('Failed to open database') from exc + ... + Traceback (most recent call last): + File "", line 2, in + File "", line 2, in func + OSError + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "", line 4, in + RuntimeError + +The expression following the :keyword:`from` must be either an exception or +``None``. Exception chaining happens automatically when an exception is raised +inside an exception handler or :keyword:`finally` section. Exception chaining +can be disabled by using ``from None`` idiom: + + >>> try: + ... open('database.sqlite') + ... except IOError: + ... raise RuntimeError from None + ... + Traceback (most recent call last): + File "", line 4, in + RuntimeError + + .. _tut-userexceptions: User-defined Exceptions @@ -341,15 +388,46 @@ example:: File "", line 2, in KeyboardInterrupt -A *finally clause* is always executed before leaving the :keyword:`try` -statement, whether an exception has occurred or not. When an exception has -occurred in the :keyword:`!try` clause and has not been handled by an -:keyword:`except` clause (or it has occurred in an :keyword:`!except` or -:keyword:`!else` clause), it is re-raised after the :keyword:`finally` clause has -been executed. The :keyword:`!finally` clause is also executed "on the way out" -when any other clause of the :keyword:`!try` statement is left via a -:keyword:`break`, :keyword:`continue` or :keyword:`return` statement. A more -complicated example:: +If a :keyword:`finally` clause is present, the :keyword:`!finally` +clause will execute as the last task before the :keyword:`try` +statement completes. The :keyword:`!finally` clause runs whether or +not the :keyword:`!try` statement produces an exception. The following +points discuss more complex cases when an exception occurs: + +* If an exception occurs during execution of the :keyword:`!try` + clause, the exception may be handled by an :keyword:`except` + clause. If the exception is not handled by an :keyword:`!except` + clause, the exception is re-raised after the :keyword:`!finally` + clause has been executed. + +* An exception could occur during execution of an :keyword:`!except` + or :keyword:`!else` clause. Again, the exception is re-raised after + the :keyword:`!finally` clause has been executed. + +* If the :keyword:`!try` statement reaches a :keyword:`break`, + :keyword:`continue` or :keyword:`return` statement, the + :keyword:`!finally` clause will execute just prior to the + :keyword:`!break`, :keyword:`!continue` or :keyword:`!return` + statement's execution. + +* If a :keyword:`!finally` clause includes a :keyword:`!return` + statement, the returned value will be the one from the + :keyword:`!finally` clause's :keyword:`!return` statement, not the + value from the :keyword:`!try` clause's :keyword:`!return` + statement. + +For example:: + + >>> def bool_return(): + ... try: + ... return True + ... finally: + ... return False + ... + >>> bool_return() + False + +A more complicated example:: >>> def divide(x, y): ... try: diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index fc2bd5578c4cf1..366a532e817afa 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -172,7 +172,7 @@ Positional and keyword arguments can be arbitrarily combined:: If you have a really long format string that you don't want to split up, it would be nice if you could reference the variables to be formatted by name instead of by position. This can be done by simply passing the dict and using -square brackets ``'[]'`` to access the keys :: +square brackets ``'[]'`` to access the keys. :: >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 8637678} >>> print('Jack: {0[Jack]:d}; Sjoerd: {0[Sjoerd]:d}; ' @@ -257,10 +257,10 @@ left with zeros. It understands about plus and minus signs:: Old string formatting --------------------- -The ``%`` operator can also be used for string formatting. It interprets the -left argument much like a :c:func:`sprintf`\ -style format string to be applied -to the right argument, and returns the string resulting from this formatting -operation. For example:: +The % operator (modulo) can also be used for string formatting. Given ``'string' +% values``, instances of ``%`` in ``string`` are replaced with zero or more +elements of ``values``. This operation is commonly known as string +interpolation. For example:: >>> import math >>> print('The value of pi is approximately %5.3f.' % math.pi) @@ -358,8 +358,8 @@ To read a file's contents, call ``f.read(size)``, which reads some quantity of data and returns it as a string (in text mode) or bytes object (in binary mode). *size* is an optional numeric argument. When *size* is omitted or negative, the entire contents of the file will be read and returned; it's your problem if the -file is twice as large as your machine's memory. Otherwise, at most *size* bytes -are read and returned. +file is twice as large as your machine's memory. Otherwise, at most *size* +characters (in text mode) or *size* bytes (in binary mode) are read and returned. If the end of the file has been reached, ``f.read()`` will return an empty string (``''``). :: @@ -412,11 +412,11 @@ or a bytes object (in binary mode) -- before writing them:: represented as number of bytes from the beginning of the file when in binary mode and an opaque number when in text mode. -To change the file object's position, use ``f.seek(offset, from_what)``. The position is computed +To change the file object's position, use ``f.seek(offset, whence)``. The position is computed from adding *offset* to a reference point; the reference point is selected by -the *from_what* argument. A *from_what* value of 0 measures from the beginning +the *whence* argument. A *whence* value of 0 measures from the beginning of the file, 1 uses the current file position, and 2 uses the end of the file as -the reference point. *from_what* can be omitted and defaults to 0, using the +the reference point. *whence* can be omitted and defaults to 0, using the beginning of the file as the reference point. :: >>> f = open('workfile', 'rb+') diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index ab92af044e7ce0..d2733a9968fb1d 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -10,25 +10,23 @@ Using the Python Interpreter Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.8` +The Python interpreter is usually installed as :file:`/usr/local/bin/python3.10` on those machines where it is available; putting :file:`/usr/local/bin` in your Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.8 + python3.10 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) -On Windows machines, the Python installation is usually placed in -:file:`C:\\Python36`, though you can change this when you're running the -installer. To add this directory to your path, you can type the following -command into :ref:`a command prompt window `:: - - set path=%path%;C:\python36 +On Windows machines where you have installed Python from the :ref:`Microsoft Store +`, the :file:`python3.10` command will be available. If you have +the :ref:`py.exe launcher ` installed, you can use the :file:`py` +command. See :ref:`setting-envvars` for other ways to launch Python. Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on Windows) at the primary prompt causes the interpreter to exit with a zero exit @@ -36,13 +34,14 @@ status. If that doesn't work, you can exit the interpreter by typing the following command: ``quit()``. The interpreter's line-editing features include interactive editing, history -substitution and code completion on systems that support readline. Perhaps the -quickest check to see whether command line editing is supported is typing -:kbd:`Control-P` to the first Python prompt you get. If it beeps, you have command -line editing; see Appendix :ref:`tut-interacting` for an introduction to the -keys. If nothing appears to happen, or if ``^P`` is echoed, command line -editing isn't available; you'll only be able to use backspace to remove -characters from the current line. +substitution and code completion on systems that support the `GNU Readline +`_ library. +Perhaps the quickest check to see whether command line editing is supported is +typing :kbd:`Control-P` to the first Python prompt you get. If it beeps, you +have command line editing; see Appendix :ref:`tut-interacting` for an +introduction to the keys. If nothing appears to happen, or if ``^P`` is +echoed, command line editing isn't available; you'll only be able to use +backspace to remove characters from the current line. The interpreter operates somewhat like the Unix shell: when called with standard input connected to a tty device, it reads and executes commands interactively; @@ -98,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.8 - Python 3.8 (default, Sep 16 2015, 09:25:04) + $ python3.10 + Python 3.10 (default, June 4 2019, 09:25:04) [GCC 4.8.2] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 3e0c99558ed7f3..2a1666128a2015 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -383,7 +383,7 @@ items of different types, but usually the items all have the same type. :: >>> squares [1, 4, 9, 16, 25] -Like strings (and all other built-in :term:`sequence` type), lists can be +Like strings (and all other built-in :term:`sequence` types), lists can be indexed and sliced:: >>> squares[0] # indexing returns the item @@ -394,7 +394,8 @@ indexed and sliced:: [9, 16, 25] All slice operations return a new list containing the requested elements. This -means that the following slice returns a new (shallow) copy of the list:: +means that the following slice returns a +:ref:`shallow copy ` of the list:: >>> squares[:] [1, 4, 9, 16, 25] diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index fd594fd97af4d4..af595e5ca04d7e 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -306,23 +306,27 @@ defines. It returns a sorted list of strings:: >>> dir(fibo) ['__name__', 'fib', 'fib2'] >>> dir(sys) # doctest: +NORMALIZE_WHITESPACE - ['__displayhook__', '__doc__', '__excepthook__', '__loader__', '__name__', - '__package__', '__stderr__', '__stdin__', '__stdout__', - '_clear_type_cache', '_current_frames', '_debugmallocstats', '_getframe', - '_home', '_mercurial', '_xoptions', 'abiflags', 'api_version', 'argv', - 'base_exec_prefix', 'base_prefix', 'builtin_module_names', 'byteorder', - 'call_tracing', 'callstats', 'copyright', 'displayhook', - 'dont_write_bytecode', 'exc_info', 'excepthook', 'exec_prefix', - 'executable', 'exit', 'flags', 'float_info', 'float_repr_style', - 'getcheckinterval', 'getdefaultencoding', 'getdlopenflags', - 'getfilesystemencoding', 'getobjects', 'getprofile', 'getrecursionlimit', - 'getrefcount', 'getsizeof', 'getswitchinterval', 'gettotalrefcount', + ['__breakpointhook__', '__displayhook__', '__doc__', '__excepthook__', + '__interactivehook__', '__loader__', '__name__', '__package__', '__spec__', + '__stderr__', '__stdin__', '__stdout__', '__unraisablehook__', + '_clear_type_cache', '_current_frames', '_debugmallocstats', '_framework', + '_getframe', '_git', '_home', '_xoptions', 'abiflags', 'addaudithook', + 'api_version', 'argv', 'audit', 'base_exec_prefix', 'base_prefix', + 'breakpointhook', 'builtin_module_names', 'byteorder', 'call_tracing', + 'callstats', 'copyright', 'displayhook', 'dont_write_bytecode', 'exc_info', + 'excepthook', 'exec_prefix', 'executable', 'exit', 'flags', 'float_info', + 'float_repr_style', 'get_asyncgen_hooks', 'get_coroutine_origin_tracking_depth', + 'getallocatedblocks', 'getdefaultencoding', 'getdlopenflags', + 'getfilesystemencodeerrors', 'getfilesystemencoding', 'getprofile', + 'getrecursionlimit', 'getrefcount', 'getsizeof', 'getswitchinterval', 'gettrace', 'hash_info', 'hexversion', 'implementation', 'int_info', - 'intern', 'maxsize', 'maxunicode', 'meta_path', 'modules', 'path', - 'path_hooks', 'path_importer_cache', 'platform', 'prefix', 'ps1', - 'setcheckinterval', 'setdlopenflags', 'setprofile', 'setrecursionlimit', - 'setswitchinterval', 'settrace', 'stderr', 'stdin', 'stdout', - 'thread_info', 'version', 'version_info', 'warnoptions'] + 'intern', 'is_finalizing', 'last_traceback', 'last_type', 'last_value', + 'maxsize', 'maxunicode', 'meta_path', 'modules', 'path', 'path_hooks', + 'path_importer_cache', 'platform', 'prefix', 'ps1', 'ps2', 'pycache_prefix', + 'set_asyncgen_hooks', 'set_coroutine_origin_tracking_depth', 'setdlopenflags', + 'setprofile', 'setrecursionlimit', 'setswitchinterval', 'settrace', 'stderr', + 'stdin', 'stdout', 'thread_info', 'unraisablehook', 'version', 'version_info', + 'warnoptions'] Without arguments, :func:`dir` lists the names you have defined currently:: @@ -523,7 +527,7 @@ Although certain modules are designed to export only names that follow certain patterns when you use ``import *``, it is still considered bad practice in production code. -Remember, there is nothing wrong with using ``from Package import +Remember, there is nothing wrong with using ``from package import specific_submodule``! In fact, this is the recommended notation unless the importing module needs to use submodules with the same name from different packages. diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 82261a6513489a..f33265cd2b0eb4 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python38' + 'C:\\Python310' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 @@ -72,9 +72,22 @@ three`` at the command line:: >>> print(sys.argv) ['demo.py', 'one', 'two', 'three'] -The :mod:`getopt` module processes *sys.argv* using the conventions of the Unix -:func:`getopt` function. More powerful and flexible command line processing is -provided by the :mod:`argparse` module. +The :mod:`argparse` module provides a more sophisticated mechanism to process +command line arguments. The following script extracts one or more filenames +and an optional number of lines to be displayed:: + + import argparse + + parser = argparse.ArgumentParser(prog = 'top', + description = 'Show top lines from each file') + parser.add_argument('filenames', nargs='+') + parser.add_argument('-l', '--lines', type=int, default=10) + args = parser.parse_args() + print(args) + +When run at the command line with ``python top.py --lines=5 alpha.txt +beta.txt``, the script sets ``args.lines`` to ``5`` and ``args.filenames`` +to ``['alpha.txt', 'beta.txt']``. .. _tut-stderr: diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index d2ac57b8e4d3a7..298034d3b48440 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -278,7 +278,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python38/lib/weakref.py", line 46, in __getitem__ + File "C:/python310/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index dc4136e42a887d..f422146aae8a30 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -50,6 +50,12 @@ This will create the ``tutorial-env`` directory if it doesn't exist, and also create directories inside it containing a copy of the Python interpreter, the standard library, and various supporting files. +A common directory location for a virtual environment is ``.venv``. +This name keeps the directory typically hidden in your shell and thus +out of the way while giving it a name that explains why the directory +exists. It also prevents clashing with ``.env`` environment variable +definition files that some tooling supports. + Once you've created a virtual environment, you may activate it. On Windows, run:: @@ -110,7 +116,7 @@ You can install the latest version of a package by specifying a package's name: .. code-block:: bash - (tutorial-env) $ pip install novas + (tutorial-env) $ python -m pip install novas Collecting novas Downloading novas-3.1.1.3.tar.gz (136kB) Installing collected packages: novas @@ -122,7 +128,7 @@ package name followed by ``==`` and the version number: .. code-block:: bash - (tutorial-env) $ pip install requests==2.6.0 + (tutorial-env) $ python -m pip install requests==2.6.0 Collecting requests==2.6.0 Using cached requests-2.6.0-py2.py3-none-any.whl Installing collected packages: requests @@ -135,7 +141,7 @@ install --upgrade`` to upgrade the package to the latest version: .. code-block:: bash - (tutorial-env) $ pip install --upgrade requests + (tutorial-env) $ python -m pip install --upgrade requests Collecting requests Installing collected packages: requests Found existing installation: requests 2.6.0 @@ -193,7 +199,7 @@ necessary packages with ``install -r``: .. code-block:: bash - (tutorial-env) $ pip install -r requirements.txt + (tutorial-env) $ python -m pip install -r requirements.txt Collecting novas==3.1.1.3 (from -r requirements.txt (line 1)) ... Collecting numpy==1.9.2 (from -r requirements.txt (line 2)) diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index d876d0740d8065..38ce9f0a900c28 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -39,11 +39,11 @@ More Python resources: * https://docs.python.org: Fast access to Python's documentation. * https://pypi.org: The Python Package Index, previously also nicknamed - the Cheese Shop, is an index of user-created Python modules that are available + the Cheese Shop [#]_, is an index of user-created Python modules that are available for download. Once you begin releasing code, you can register it here so that others can find it. -* https://code.activestate.com/recipes/langs/python/: The Python Cookbook is a +* https://github.com/ActiveState/code/tree/master/recipes/Python: The Python Cookbook is a sizable collection of code examples, larger modules, and useful scripts. Particularly notable contributions are collected in a book also titled Python Cookbook (O'Reilly & Associates, ISBN 0-596-00797-3.) @@ -68,3 +68,9 @@ Before posting, be sure to check the list of :ref:`Frequently Asked Questions ` (also called the FAQ). The FAQ answers many of the questions that come up again and again, and may already contain the solution for your problem. + +.. rubric:: Footnotes + +.. [#] "Cheese Shop" is a Monty Python's sketch: a customer enters a cheese shop, + but whatever cheese he asks for, the clerk says it's missing. + diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 0574336cf35403..8c65d99ef31f9f 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -1,4 +1,4 @@ -.. highlightlang:: sh +.. highlight:: sh .. ATTENTION: You probably should update Misc/python.man, too, if you modify this file. @@ -70,6 +70,7 @@ source. :data:`sys.path` (allowing modules in that directory to be imported as top level modules). + .. audit-event:: cpython.run_command command cmdoption-c .. cmdoption:: -m @@ -100,11 +101,18 @@ source. first element will be set to ``"-m"``). As with the :option:`-c` option, the current directory will be added to the start of :data:`sys.path`. + :option:`-I` option can be used to run the script in isolated mode where + :data:`sys.path` contains neither the current directory nor the user's + site-packages directory. All :envvar:`PYTHON*` environment variables are + ignored, too. + Many standard library modules contain code that is invoked on their execution as a script. An example is the :mod:`timeit` module:: - python -mtimeit -s 'setup here' 'benchmarked code here' - python -mtimeit -h # for details + python -m timeit -s 'setup here' 'benchmarked code here' + python -m timeit -h # for details + + .. audit-event:: cpython.run_module module-name cmdoption-m .. seealso:: :func:`runpy.run_module` @@ -112,13 +120,13 @@ source. :pep:`338` -- Executing modules as scripts - .. versionchanged:: 3.1 Supply the package name to run a ``__main__`` submodule. .. versionchanged:: 3.4 namespace packages are also supported +.. _cmdarg-dash: .. describe:: - @@ -129,6 +137,9 @@ source. ``"-"`` and the current directory will be added to the start of :data:`sys.path`. + .. audit-event:: cpython.run_stdin "" "" + +.. _cmdarg-script: .. describe::
    • FormatPacked as .zipPacked as .tar.bz2