diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 1576599379c489..fcfac85ed9c442 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -59,7 +59,7 @@ jobs: variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.1b + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml @@ -116,7 +116,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1b + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml index 492e4e34bb2dab..96361961ea75eb 100644 --- a/.azure-pipelines/docs-steps.yml +++ b/.azure-pipelines/docs-steps.yml @@ -12,7 +12,7 @@ steps: inputs: versionSpec: '>=3.6' -- script: python -m pip install sphinx==1.8.2 blurb python-docs-theme +- script: python -m pip install sphinx==2.0.1 blurb python-docs-theme displayName: 'Install build dependencies' - ${{ if ne(parameters.latex, 'true') }}: diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 0bd7921bcbefcc..2486f88a63fb15 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -59,7 +59,7 @@ jobs: variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.0j + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml @@ -116,7 +116,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.0j + openssl_version: 1.1.1c steps: - template: ./posix-steps.yml diff --git a/.azure-pipelines/windows-release.yml b/.azure-pipelines/windows-release.yml new file mode 100644 index 00000000000000..774585792484dd --- /dev/null +++ b/.azure-pipelines/windows-release.yml @@ -0,0 +1,96 @@ +name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) + +# QUEUE TIME VARIABLES +# variables: +# GitRemote: python +# SourceTag: +# DoPGO: true +# SigningCertificate: 'Python Software Foundation' +# SigningDescription: 'Built: $(Build.BuildNumber)' +# DoLayout: true +# DoMSIX: true +# DoNuget: true +# DoEmbed: true +# DoMSI: true +# DoPublish: false + +trigger: none +pr: none + +stages: +- stage: Build + displayName: Build binaries + jobs: + - template: windows-release/stage-build.yml + +- stage: Sign + displayName: Sign binaries + dependsOn: Build + jobs: + - template: windows-release/stage-sign.yml + +- stage: Layout + displayName: Generate layouts + dependsOn: Sign + jobs: + - template: windows-release/stage-layout-full.yml + - template: windows-release/stage-layout-embed.yml + - template: windows-release/stage-layout-nuget.yml + +- stage: Pack + dependsOn: Layout + jobs: + - template: windows-release/stage-pack-nuget.yml + +- stage: Test + dependsOn: Pack + jobs: + - template: windows-release/stage-test-embed.yml + - template: windows-release/stage-test-nuget.yml + +- stage: Layout_MSIX + displayName: Generate MSIX layouts + dependsOn: Sign + condition: and(succeeded(), eq(variables['DoMSIX'], 'true')) + jobs: + - template: windows-release/stage-layout-msix.yml + +- stage: Pack_MSIX + displayName: Package MSIX + dependsOn: Layout_MSIX + jobs: + - template: windows-release/stage-pack-msix.yml + +- stage: Build_MSI + displayName: Build MSI installer + dependsOn: Sign + condition: and(succeeded(), eq(variables['DoMSI'], 'true')) + jobs: + - template: windows-release/stage-msi.yml + +- stage: Test_MSI + displayName: Test MSI installer + dependsOn: Build_MSI + jobs: + - template: windows-release/stage-test-msi.yml + +- stage: PublishPyDotOrg + displayName: Publish to python.org + dependsOn: ['Test_MSI', 'Test'] + condition: and(succeeded(), eq(variables['DoPublish'], 'true')) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishNuget + displayName: Publish to nuget.org + dependsOn: Test + condition: and(succeeded(), eq(variables['DoPublish'], 'true')) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishStore + displayName: Publish to Store + dependsOn: Pack_MSIX + condition: and(succeeded(), eq(variables['DoPublish'], 'true')) + jobs: + - template: windows-release/stage-publish-store.yml diff --git a/.azure-pipelines/windows-release/build-steps.yml b/.azure-pipelines/windows-release/build-steps.yml new file mode 100644 index 00000000000000..508d73b0865fee --- /dev/null +++ b/.azure-pipelines/windows-release/build-steps.yml @@ -0,0 +1,83 @@ +parameters: + ShouldPGO: false + +steps: +- template: ./checkout.yml + +- powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)" + displayName: 'Extract version numbers' + +- ${{ if eq(parameters.ShouldPGO, 'false') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration) + displayName: 'Run build' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- ${{ if eq(parameters.ShouldPGO, 'true') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) --pgo + displayName: 'Run build with PGO' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + displayName: 'Add WinSDK tools to path' + +- powershell: | + $env:SigningCertificate = $null + .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default + makecat "${env:CAT}.cdf" + del "${env:CAT}.cdf" + if (-not (Test-Path "${env:CAT}.cat")) { + throw "Failed to build catalog file" + } + displayName: 'Generate catalog' + env: + CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python + +- task: PublishBuildArtifacts@1 + displayName: 'Publish binaries' + condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))) + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\bin\$(Arch)' + ArtifactName: bin_$(Name) + +- task: PublishBuildArtifacts@1 + displayName: 'Publish binaries for signing' + condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])) + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\bin\$(Arch)' + ArtifactName: unsigned_bin_$(Name) + +- task: CopyFiles@2 + displayName: 'Layout Artifact: symbols' + inputs: + sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch) + targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name) + flatten: true + contents: | + **\*.pdb + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: symbols' + inputs: + PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols' + ArtifactName: symbols diff --git a/.azure-pipelines/windows-release/checkout.yml b/.azure-pipelines/windows-release/checkout.yml new file mode 100644 index 00000000000000..d42d55fff08dda --- /dev/null +++ b/.azure-pipelines/windows-release/checkout.yml @@ -0,0 +1,21 @@ +parameters: + depth: 3 + +steps: +- checkout: none + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/$(SourceTag))' + condition: and(succeeded(), and(variables['GitRemote'], variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone (/$(SourceTag))' + condition: and(succeeded(), and(not(variables['GitRemote']), variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/)' + condition: and(succeeded(), and(variables['GitRemote'], not(variables['SourceTag']))) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone' + condition: and(succeeded(), and(not(variables['GitRemote']), not(variables['SourceTag']))) diff --git a/.azure-pipelines/windows-release/find-sdk.yml b/.azure-pipelines/windows-release/find-sdk.yml new file mode 100644 index 00000000000000..e4de78555b3f66 --- /dev/null +++ b/.azure-pipelines/windows-release/find-sdk.yml @@ -0,0 +1,17 @@ +# Locate the Windows SDK and add its binaries directory to PATH +# +# `toolname` can be overridden to use a different marker file. + +parameters: + toolname: signtool.exe + +steps: + - powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\${{ parameters.toolname }}" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + Write-Host "Adding $($tool.Directory) to PATH" + displayName: 'Add WinSDK tools to path' diff --git a/.azure-pipelines/windows-release/layout-command.yml b/.azure-pipelines/windows-release/layout-command.yml new file mode 100644 index 00000000000000..3ec9b69ad7121c --- /dev/null +++ b/.azure-pipelines/windows-release/layout-command.yml @@ -0,0 +1,20 @@ +steps: +- powershell: > + Write-Host ( + '##vso[task.setvariable variable=LayoutCmd]& + "{0}" + "{1}\PC\layout" + -vv + --source "{1}" + --build "{2}" + --temp "{3}" + --include-cat "{2}\python.cat" + --doc-build "{4}"' + -f ( + "$(PYTHON)", + "$(Build.SourcesDirectory)", + (Split-Path -Parent "$(PYTHON)"), + "$(Build.BinariesDirectory)\layout-temp", + "$(Build.BinariesDirectory)\doc" + )) + displayName: 'Set LayoutCmd' diff --git a/.azure-pipelines/windows-release/mingw-lib.yml b/.azure-pipelines/windows-release/mingw-lib.yml new file mode 100644 index 00000000000000..30f7d34fa61d23 --- /dev/null +++ b/.azure-pipelines/windows-release/mingw-lib.yml @@ -0,0 +1,13 @@ +parameters: + DllToolOpt: -m i386:x86-64 + #DllToolOpt: -m i386 --as-flags=--32 + +steps: +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch binutils --single-branch --depth 1 --progress -v "binutils" + gci "bin\$(Arch)\python*.dll" | %{ + & "binutils\gendef.exe" $_ | Out-File -Encoding ascii tmp.def + & "binutils\dlltool.exe" --dllname $($_.BaseName).dll --def tmp.def --output-lib "$($_.Directory)\lib$($_.BaseName).a" ${{ parameters.DllToolOpt }} + } + displayName: 'Generate MinGW import library' + workingDirectory: $(Build.BinariesDirectory) diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml new file mode 100644 index 00000000000000..153408271c711e --- /dev/null +++ b/.azure-pipelines/windows-release/msi-steps.yml @@ -0,0 +1,146 @@ +steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + downloadPath: $(Build.BinariesDirectory) + + - task: CopyFiles@2 + displayName: 'Merge documentation files' + inputs: + sourceFolder: $(Build.BinariesDirectory)\doc + targetFolder: $(Build.SourcesDirectory)\Doc\build + contents: | + htmlhelp\*.chm + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_win32' + inputs: + artifactName: bin_win32 + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_win32_d' + inputs: + artifactName: bin_win32_d + downloadPath: $(Build.BinariesDirectory) + + - task: CopyFiles@2 + displayName: 'Merge win32 debug files' + inputs: + sourceFolder: $(Build.BinariesDirectory)\bin_win32_d + targetFolder: $(Build.BinariesDirectory)\bin_win32 + contents: | + **\*_d.* + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_amd64' + inputs: + artifactName: bin_amd64 + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_amd64_d' + inputs: + artifactName: bin_amd64_d + downloadPath: $(Build.BinariesDirectory) + + - task: CopyFiles@2 + displayName: 'Merge amd64 debug files' + inputs: + sourceFolder: $(Build.BinariesDirectory)\bin_amd64_d + targetFolder: $(Build.BinariesDirectory)\bin_amd64 + contents: | + **\*_d.* + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: tcltk_lib_win32' + inputs: + artifactName: tcltk_lib_win32 + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: tcltk_lib_amd64' + inputs: + artifactName: tcltk_lib_amd64 + downloadPath: $(Build.BinariesDirectory) + + - script: | + ren bin_win32 win32 + ren bin_amd64 amd64 + displayName: 'Correct artifact directory names' + workingDirectory: $(Build.BinariesDirectory) + + - script: | + call Tools\msi\get_externals.bat + call PCbuild\find_python.bat + echo ##vso[task.setvariable variable=PYTHON]%PYTHON% + call PCbuild/find_msbuild.bat + echo ##vso[task.setvariable variable=MSBUILD]%MSBUILD% + displayName: 'Get external dependencies' + + - script: | + %PYTHON% -m pip install blurb + %PYTHON% -m blurb merge -f Misc\NEWS + displayName: 'Merge NEWS file' + + - script: | + %MSBUILD% Tools\msi\launcher\launcher.wixproj + displayName: 'Build launcher installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false + displayName: 'Build win32 installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\win32\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_win32 + BuildForRelease: true + SuppressMinGWLib: true + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false + displayName: 'Build amd64 installer' + env: + Platform: x64 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_amd64 + BuildForRelease: true + SuppressMinGWLib: true + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (1/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\win32\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\win32 + contents: | + *.msi + *.cab + *.exe + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (2/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\amd64\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\amd64 + contents: | + *.msi + *.cab + *.exe + + - task: PublishBuildArtifacts@1 + displayName: 'Publish MSI' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msi' + ArtifactName: msi diff --git a/.azure-pipelines/windows-release/stage-build.yml b/.azure-pipelines/windows-release/stage-build.yml new file mode 100644 index 00000000000000..a5093a04f08716 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-build.yml @@ -0,0 +1,160 @@ +jobs: +- job: Build_Docs + displayName: Docs build + pool: + name: 'Windows Release' + #vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: Doc\make.bat html + displayName: 'Build HTML docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + #- powershell: iwr "https://www.python.org/ftp/python/3.7.3/python373.chm" -OutFile "$(Build.BinariesDirectory)\python390a0.chm" + # displayName: 'Cheat at building CHM docs' + + - script: Doc\make.bat htmlhelp + displayName: 'Build CHM docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + - task: CopyFiles@2 + displayName: 'Assemble artifact: Doc' + inputs: + sourceFolder: $(Build.BinariesDirectory)\Doc + targetFolder: $(Build.ArtifactStagingDirectory)\Doc + contents: | + html\**\* + htmlhelp\*.chm + + - task: PublishBuildArtifacts@1 + displayName: 'Publish artifact: doc' + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory)\Doc + ArtifactName: doc + +- job: Build_Python + displayName: Python build + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Arch: win32 + Platform: x86 + Configuration: Release + win32_d: + Name: win32_d + Arch: win32 + Platform: x86 + Configuration: Debug + amd64_d: + Name: amd64_d + Arch: amd64 + Platform: x64 + Configuration: Debug + + steps: + - template: ./build-steps.yml + +- job: Build_Python_NonPGO + displayName: Python non-PGO build + condition: and(succeeded(), ne(variables['DoPGO'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + + steps: + - template: ./build-steps.yml + + +- job: Build_Python_PGO + displayName: Python PGO build + condition: and(succeeded(), eq(variables['DoPGO'], 'true')) + + # Allow up to five hours for PGO + timeoutInMinutes: 300 + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + + steps: + - template: ./build-steps.yml + parameters: + ShouldPGO: true + + +- job: TclTk_Lib + displayName: Publish Tcl/Tk Library + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: PCbuild\get_externals.bat --no-openssl --no-libffi + displayName: 'Get external dependencies' + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x86 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_win32" + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x64 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_amd64" + + - task: PublishBuildArtifacts@1 + displayName: 'Publish artifact: tcltk_lib_win32' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\tcl_win32' + ArtifactName: tcltk_lib_win32 + + - task: PublishBuildArtifacts@1 + displayName: 'Publish artifact: tcltk_lib_amd64' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\tcl_amd64' + ArtifactName: tcltk_lib_amd64 diff --git a/.azure-pipelines/windows-release/stage-layout-embed.yml b/.azure-pipelines/windows-release/stage-layout-embed.yml new file mode 100644 index 00000000000000..e2689dbb603d6e --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-embed.yml @@ -0,0 +1,56 @@ +jobs: +- job: Make_Embed_Layout + displayName: Make embeddable layout + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - template: ./layout-command.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + displayName: 'Extract version numbers' + + - powershell: > + $(LayoutCmd) + --copy "$(Build.ArtifactStagingDirectory)\layout" + --zip "$(Build.ArtifactStagingDirectory)\embed\python-$(VersionText)-embed-$(Name).zip" + --preset-embed + displayName: 'Generate embeddable layout' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: layout_embed_$(Name)' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\layout' + ArtifactName: layout_embed_$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: embed' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\embed' + ArtifactName: embed diff --git a/.azure-pipelines/windows-release/stage-layout-full.yml b/.azure-pipelines/windows-release/stage-layout-full.yml new file mode 100644 index 00000000000000..3593cf0a3f6943 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-full.yml @@ -0,0 +1,62 @@ +jobs: +- job: Make_Layouts + displayName: Make layouts + condition: and(succeeded(), eq(variables['DoLayout'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: tcltk_lib_$(Name)' + inputs: + artifactName: tcltk_lib_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default + displayName: 'Generate full layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8 + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: layout_full_$(Name)' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\layout' + ArtifactName: layout_full_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-msix.yml b/.azure-pipelines/windows-release/stage-layout-msix.yml new file mode 100644 index 00000000000000..1a1e0a2fd68510 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-msix.yml @@ -0,0 +1,86 @@ +jobs: +- job: Make_MSIX_Layout + displayName: Make MSIX layout + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + #win32: + # Name: win32 + # Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + # PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: tcltk_lib_$(Name)' + inputs: + artifactName: tcltk_lib_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - template: ./layout-command.yml + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx-store" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile + displayName: 'Generate store APPX layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8 + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: layout_appxstore_$(Name)' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\appx-store' + ArtifactName: layout_appxstore_$(Name) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: cert' + condition: and(succeeded(), variables['SigningCertificate']) + inputs: + artifactName: cert + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $info = (gc "$(Build.BinariesDirectory)\cert\certinfo.json" | ConvertFrom-JSON) + Write-Host "Side-loadable APPX must be signed with '$($info.Subject)'" + Write-Host "##vso[task.setvariable variable=APPX_DATA_PUBLISHER]$($info.Subject)" + Write-Host "##vso[task.setvariable variable=APPX_DATA_SHA256]$($info.SHA256)" + displayName: 'Override signing parameters' + condition: and(succeeded(), variables['SigningCertificate']) + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests + displayName: 'Generate sideloading APPX layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8 + APPX_DATA_PUBLISHER: $(APPX_DATA_PUBLISHER) + APPX_DATA_SHA256: $(APPX_DATA_SHA256) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: layout_appx_$(Name)' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\appx' + ArtifactName: layout_appx_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-nuget.yml b/.azure-pipelines/windows-release/stage-layout-nuget.yml new file mode 100644 index 00000000000000..ca4213d9e5c2e8 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-nuget.yml @@ -0,0 +1,44 @@ +jobs: +- job: Make_Nuget_Layout + displayName: Make Nuget layout + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget + displayName: 'Generate nuget layout' + env: + TCL_LIBRARY: $(Build.BinariesDirectory)\bin_$(Name)\tcl\tcl8 + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: layout_nuget_$(Name)' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\nuget' + ArtifactName: layout_nuget_$(Name) diff --git a/.azure-pipelines/windows-release/stage-msi.yml b/.azure-pipelines/windows-release/stage-msi.yml new file mode 100644 index 00000000000000..7afc816a0c6e9c --- /dev/null +++ b/.azure-pipelines/windows-release/stage-msi.yml @@ -0,0 +1,36 @@ +jobs: +- job: Make_MSI + displayName: Make MSI + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmName: win2016-vs2017 + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml + +- job: Make_Signed_MSI + displayName: Make signed MSI + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml new file mode 100644 index 00000000000000..6f1846e581ef13 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -0,0 +1,127 @@ +jobs: +- job: Pack_MSIX + displayName: Pack MSIX bundles + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Artifact: appx + Suffix: + ShouldSign: true + amd64_store: + Name: amd64 + Artifact: appxstore + Suffix: -store + Upload: true + + steps: + - template: ./checkout.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: layout_$(Artifact)_$(Name)' + inputs: + artifactName: layout_$(Artifact)_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: symbols' + inputs: + artifactName: symbols + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[task.setvariable variable=Filename]python-$($d.PythonVersion)-$(Name)$(Suffix)" + displayName: 'Extract version numbers' + + - powershell: | + ./Tools/msi/make_appx.ps1 -layout "$(Build.BinariesDirectory)\layout_$(Artifact)_$(Name)" -msix "$(Build.ArtifactStagingDirectory)\msix\$(Filename).msix" + displayName: 'Build msix' + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msix\$(Filename).appxsym" *.pdb + displayName: 'Build appxsym' + workingDirectory: $(Build.BinariesDirectory)\symbols\$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), or(ne(variables['ShouldSign'], 'true'), not(variables['SigningCertificate']))) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), and(eq(variables['ShouldSign'], 'true'), variables['SigningCertificate'])) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: unsigned_msix + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msixupload\$(Filename).msixupload" * + displayName: 'Build msixupload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + workingDirectory: $(Build.ArtifactStagingDirectory)\msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIXUpload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msixupload' + ArtifactName: msixupload + + +- job: Sign_MSIX + displayName: Sign side-loadable MSIX bundles + dependsOn: + - Pack_MSIX + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + steps: + - checkout: none + - template: ./find-sdk.yml + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Artifact: unsigned_msix' + inputs: + artifactName: unsigned_msix + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $failed = $true + foreach ($retry in 1..3) { + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix) + if ($?) { + $failed = $false + break + } + sleep 1 + } + if ($failed) { + throw "Failed to sign MSIX" + } + displayName: 'Sign MSIX' + workingDirectory: $(Build.BinariesDirectory)\unsigned_msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\unsigned_msix' + ArtifactName: msix diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml new file mode 100644 index 00000000000000..5aa394fa48a1c7 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -0,0 +1,41 @@ +jobs: +- job: Pack_Nuget + displayName: Pack Nuget bundles + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + win32: + Name: win32 + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: layout_nuget_$(Name)' + inputs: + artifactName: layout_nuget_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - task: NugetToolInstaller@0 + displayName: 'Install Nuget' + inputs: + versionSpec: '>=5.0' + + - powershell: | + nuget pack "$(Build.BinariesDirectory)\layout_nuget_$(Name)\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive + displayName: 'Create nuget package' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: nuget' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: nuget diff --git a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml new file mode 100644 index 00000000000000..7586d850f34095 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml @@ -0,0 +1,28 @@ +jobs: +- job: Publish_Nuget + displayName: Publish Nuget packages + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: NuGetCommand@2 + displayName: Push packages + condition: and(succeeded(), eq(variables['SigningCertificate'], 'Python Software Foundation')) + inputs: + command: push + packagesToPush: $(Build.BinariesDirectory)\nuget\*.nupkg' + nuGetFeedType: external + publishFeedCredentials: 'Python on Nuget' diff --git a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml new file mode 100644 index 00000000000000..2215a56d4bc256 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml @@ -0,0 +1,34 @@ +jobs: +- job: Publish_Python + displayName: Publish python.org packages + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true'))) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: Doc' + inputs: + artifactName: Doc + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msi' + inputs: + artifactName: msi + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + # TODO: eq(variables['SigningCertificate'], 'Python Software Foundation') + # If we are not real-signed, DO NOT PUBLISH diff --git a/.azure-pipelines/windows-release/stage-publish-store.yml b/.azure-pipelines/windows-release/stage-publish-store.yml new file mode 100644 index 00000000000000..06884c4f35b7b7 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-store.yml @@ -0,0 +1,22 @@ +jobs: +- job: Publish_Store + displayName: Publish Store packages + condition: and(succeeded(), eq(variables['DoMSIX'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + + # TODO: eq(variables['SigningCertificate'], 'Python Software Foundation') + # If we are not real-signed, DO NOT PUBLISH diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml new file mode 100644 index 00000000000000..3d6ca9457f1c7f --- /dev/null +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -0,0 +1,113 @@ +jobs: +- job: Sign_Python + displayName: Sign Python binaries + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + + steps: + - checkout: none + - template: ./find-sdk.yml + + - powershell: | + Write-Host "##vso[build.addbuildtag]signed" + displayName: 'Add build tags' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: unsigned_bin_$(Name)' + inputs: + artifactName: unsigned_bin_$(Name) + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $files = (gi *.exe, *.dll, *.pyd, *.cat -Exclude vcruntime*, libffi*, libcrypto*, libssl*) + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /d "$(SigningDescription)" $files + displayName: 'Sign binaries' + workingDirectory: $(Build.BinariesDirectory)\unsigned_bin_$(Name) + + - powershell: | + $files = (gi *.exe, *.dll, *.pyd, *.cat -Exclude vcruntime*, libffi*, libcrypto*, libssl*) + $failed = $true + foreach ($retry in 1..10) { + signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files + if ($?) { + $failed = $false + break + } + sleep 5 + } + if ($failed) { + Write-Host "##vso[task.logissue type=error]Failed to timestamp files" + } + displayName: 'Timestamp binaries' + workingDirectory: $(Build.BinariesDirectory)\unsigned_bin_$(Name) + continueOnError: true + + - task: PublishBuildArtifacts@1 + displayName: 'Publish artifact: bin_$(Name)' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\unsigned_bin_$(Name)' + ArtifactName: bin_$(Name) + + +- job: Dump_CertInfo + displayName: Capture certificate info + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + steps: + - checkout: none + + - powershell: | + $m = 'CN=$(SigningCertificate)' + $c = ((gci Cert:\CurrentUser\My), (gci Cert:\LocalMachine\My)) | %{ $_ } | ` + ?{ $_.Subject -match $m } | ` + select -First 1 + if (-not $c) { + Write-Host "Failed to find certificate for $(SigningCertificate)" + exit + } + $d = mkdir "$(Build.BinariesDirectory)\tmp" -Force + $cf = "$d\cert.cer" + [IO.File]::WriteAllBytes($cf, $c.Export("Cer")) + $csha = (certutil -dump $cf | sls "Cert Hash\(sha256\): (.+)").Matches.Groups[1].Value + + $info = @{ Subject=$c.Subject; SHA256=$csha; } + + $d = mkdir "$(Build.BinariesDirectory)\cert" -Force + $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" + displayName: "Extract certificate info" + + - task: PublishBuildArtifacts@1 + displayName: 'Publish artifact: cert' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\cert' + ArtifactName: cert + + +- job: Mark_Unsigned + displayName: Tag unsigned build + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmName: win2016-vs2017 + + steps: + - checkout: none + + - powershell: | + Write-Host "##vso[build.addbuildtag]unsigned" + displayName: 'Add build tag' diff --git a/.azure-pipelines/windows-release/stage-test-embed.yml b/.azure-pipelines/windows-release/stage-test-embed.yml new file mode 100644 index 00000000000000..b33176266a20da --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-embed.yml @@ -0,0 +1,41 @@ +jobs: +- job: Test_Embed + displayName: Test Embed + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\embed\python*embed-$(Name).zip" + Expand-Archive -Path $p -DestinationPath "$(Build.BinariesDirectory)\Python" + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Install Python and add to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' diff --git a/.azure-pipelines/windows-release/stage-test-msi.yml b/.azure-pipelines/windows-release/stage-test-msi.yml new file mode 100644 index 00000000000000..10039295a18407 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-msi.yml @@ -0,0 +1,108 @@ +jobs: +- job: Test_MSI + displayName: Test MSI + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32_User: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_User + InstallAllUsers: 0 + win32_Machine: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_Machine + InstallAllUsers: 1 + amd64_User: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_User + InstallAllUsers: 0 + amd64_Machine: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_Machine + InstallAllUsers: 1 + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msi' + inputs: + artifactName: msi + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $p = (gci -r *.exe | ?{ $_.Name -match '$(ExeMatch)' } | select -First 1) + Write-Host "##vso[task.setvariable variable=SetupExe]$($p.FullName)" + Write-Host "##vso[task.setvariable variable=SetupExeName]$($p.Name)" + displayName: 'Find installer executable' + workingDirectory: $(Build.BinariesDirectory)\msi + + - script: > + "$(SetupExe)" + /passive + /log "$(Logs)\install\log.txt" + TargetDir="$(Build.BinariesDirectory)\Python" + Include_debug=1 + Include_symbols=1 + InstallAllUsers=$(InstallAllUsers) + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - powershell: | + gci -r "${env:PROGRAMDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-machine Start Menu items' + - powershell: | + gci -r "${env:APPDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-user Start Menu items' + + - powershell: | + gci -r "HKLM:\Software\WOW6432Node\Python" + displayName: 'Capture per-machine 32-bit registry' + - powershell: | + gci -r "HKLM:\Software\Python" + displayName: 'Capture per-machine native registry' + - powershell: | + gci -r "HKCU:\Software\Python" + displayName: 'Capture current-user registry' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' + + - script: | + python -m test -uall -v test_ttk_guionly test_tk test_idle + displayName: 'Test Tkinter and Idle' + + - script: > + "$(SetupExe)" + /passive + /uninstall + /log "$(Logs)\uninstall\log.txt" + displayName: 'Uninstall Python' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: logs' + condition: true + continueOnError: true + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\logs' + ArtifactName: msi_testlogs diff --git a/.azure-pipelines/windows-release/stage-test-nuget.yml b/.azure-pipelines/windows-release/stage-test-nuget.yml new file mode 100644 index 00000000000000..1f8b601d0d023b --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-nuget.yml @@ -0,0 +1,58 @@ +jobs: +- job: Test_Nuget + displayName: Test Nuget + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmName: win2016-vs2017 + + workspace: + clean: all + + strategy: + matrix: + win32: + Package: pythonx86 + amd64: + Package: python + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: NugetToolInstaller@0 + inputs: + versionSpec: '>= 5' + + - powershell: > + nuget install + $(Package) + -Source "$(Build.BinariesDirectory)\nuget" + -OutputDirectory "$(Build.BinariesDirectory)\install" + -Prerelease + -ExcludeVersion + -NonInteractive + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\install\$(Package)\tools\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fae5138435665e..963ab4dcff425b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,8 +14,14 @@ Objects/dict* @methane # Hashing -**/*hashlib* @python/crypto-team -**/*pyhash* @python/crypto-team +**/*hashlib* @python/crypto-team @tiran +**/*pyhash* @python/crypto-team @tiran +**/*sha* @python/crypto-team @tiran +**/*md5* @python/crypto-team @tiran +**/*blake* @python/crypto-team @tiran +/Modules/_blake2/** @python/crypto-team @tiran +/Modules/_sha3/** @python/crypto-team @tiran + # HTML /Lib/html/ @ezio-melotti @@ -31,10 +37,11 @@ Objects/dict* @methane # SSL -**/*ssl* @python/crypto-team +**/*ssl* @python/crypto-team @tiran +**/*.pem @python/crypto-team @tiran # CSPRNG -Python/bootstrap_hash.c @python/crypto-team +Python/bootstrap_hash.c @python/crypto-team @tiran # Email and related **/*mail* @python/email-team diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 00000000000000..28aea946623cc5 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,18 @@ +# Security Policy + +## Supported Versions + +The Python team applies security fixes according to the table +in [the devguide]( +https://devguide.python.org/#status-of-python-branches +). + +## Reporting a Vulnerability + +Please read the guidelines on reporting security issues [on the +official website]( +https://www.python.org/news/security/#reporting-security-issues-in-python +) for instructions on how to report a security-related problem to +the Python team responsibly. + +To reach the response team, email `security at python dot org`. diff --git a/.github/appveyor.yml b/.github/appveyor.yml index e8012f69ee5b2e..8556cf8437e509 100644 --- a/.github/appveyor.yml +++ b/.github/appveyor.yml @@ -1,4 +1,4 @@ -version: 3.8build{build} +version: 3.9build{build} clone_depth: 5 branches: only: diff --git a/.gitignore b/.gitignore index e3c5809579239c..9445ef1e2c5252 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ Lib/test/data/* Makefile Makefile.pre Misc/python.pc +Misc/python-embed.pc Misc/python-config.sh Modules/Setup Modules/Setup.config diff --git a/.travis.yml b/.travis.yml index c1efe24b646bb5..addff773347977 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ cache: env: global: - - OPENSSL=1.1.0i + - OPENSSL=1.1.1c - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}" - PATH="${OPENSSL_DIR}/bin:$PATH" # Use -O3 because we don't use debugger on Travis-CI @@ -55,7 +55,7 @@ matrix: # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. # (Updating the version is fine as long as no warnings are raised by doing so.) # The theme used by the docs is stored separately, so we need to install that as well. - - python -m pip install sphinx==1.8.2 blurb python-docs-theme + - python -m pip install sphinx==2.0.1 blurb python-docs-theme script: - make check suspicious html SPHINXOPTS="-q -W -j4" - name: "Documentation tests" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index ed15b520548c74..c5f24abe2ca661 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,11 +2,11 @@ Please note that all interactions on [Python Software Foundation](https://www.python.org/psf-landing/)-supported -infrastructure is [covered](https://www.python.org/psf/records/board/minutes/2014-01-06/#management-of-the-psfs-web-properties>) +infrastructure is [covered](https://www.python.org/psf/records/board/minutes/2014-01-06/#management-of-the-psfs-web-properties) by the [PSF Code of Conduct](https://www.python.org/psf/codeofconduct/), -which includes all infrastructure used in the development of Python itself +which includes all the infrastructure used in the development of Python itself (e.g. mailing lists, issue trackers, GitHub, etc.). -In general this means everyone is expected to be open, considerate, and -respectful of others no matter what their position is within the project. +In general, this means that everyone is expected to be **open**, **considerate**, and +**respectful** of others no matter what their position is within the project. diff --git a/Doc/Makefile b/Doc/Makefile index 6f86728ea834c3..3bcd9f23752b00 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -133,7 +133,7 @@ clean: venv: $(PYTHON) -m venv $(VENVDIR) $(VENVDIR)/bin/python3 -m pip install -U pip setuptools - $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb python-docs-theme + $(VENVDIR)/bin/python3 -m pip install -U Sphinx==2.0.1 blurb python-docs-theme @echo "The venv has been created in the $(VENVDIR) directory" dist: diff --git a/Doc/README.rst b/Doc/README.rst index 31f8a8b7f59328..380ea4fa9b26ad 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -113,6 +113,15 @@ Then, from the ``Doc`` directory, run :: where ```` is one of html, text, latex, or htmlhelp (for explanations see the make targets above). +Deprecation header +================== + +You can define the ``outdated`` variable in ``html_context`` to show a +red banner on each page redirecting to the "latest" version. + +The link points to the same page on ``/3/``, sadly for the moment the +language is lost during the process. + Contributing ============ diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 27d3f76d7a3dd5..48428109e6f2fe 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -33,13 +33,19 @@ bound into a function. Return the number of free variables in *co*. -.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) +.. c:function:: PyCodeObject* PyCode_New(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) Return a new code object. If you need a dummy code object to create a frame, use :c:func:`PyCode_NewEmpty` instead. Calling :c:func:`PyCode_New` directly can bind you to a precise Python version since the definition of the bytecode changes often. + .. versionchanged:: 3.8 + An extra parameter is required (*posonlyargcount*) to support :PEP:`570`. + The first parameter (*argcount*) now represents the total number of positional arguments, + including positional-only. + + .. audit-event:: code.__new__ "code filename name argcount posonlyargcount kwonlyargcount nlocals stacksize flags" .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 675bd013e892f5..06dbb2572725ee 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -129,4 +129,10 @@ Complex Numbers as Python Objects If *op* is not a Python complex number object but has a :meth:`__complex__` method, this method will first be called to convert *op* to a Python complex - number object. Upon failure, this method returns ``-1.0`` as a real value. + number object. If ``__complex__()`` is not defined then it falls back to + :meth:`__float__`. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. Upon failure, this method returns ``-1.0`` as a real + value. + + .. versionchanged:: 3.8 + Use :meth:`__index__` if available. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 00ef00526bc5f8..d3f6daa8347e41 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -72,6 +72,9 @@ Printing and clearing .. c:function:: void PyErr_WriteUnraisable(PyObject *obj) + Call :func:`sys.unraisablehook` using the current exception and *obj* + argument. + This utility function prints a warning message to ``sys.stderr`` when an exception has been set but it is impossible for the interpreter to actually raise the exception. It is used, for example, when an exception occurs in an @@ -81,6 +84,8 @@ Printing and clearing in which the unraisable exception occurred. If possible, the repr of *obj* will be printed in the warning message. + An exception must be set when calling this function. + Raising exceptions ================== @@ -514,13 +519,13 @@ Signal Handling single: SIGINT single: KeyboardInterrupt (built-in exception) - This function simulates the effect of a :const:`SIGINT` signal arriving --- the - next time :c:func:`PyErr_CheckSignals` is called, :exc:`KeyboardInterrupt` will - be raised. It may be called without holding the interpreter lock. - - .. % XXX This was described as obsolete, but is used in - .. % _thread.interrupt_main() (used from IDLE), so it's still needed. + Simulate the effect of a :const:`SIGINT` signal arriving. The next time + :c:func:`PyErr_CheckSignals` is called, the Python signal handler for + :const:`SIGINT` will be called. + If :const:`SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. .. c:function:: int PySignal_SetWakeupFd(int fd) diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index defc859dd5c478..543dc60b9f55ab 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -60,6 +60,32 @@ the :mod:`io` APIs instead. raised if the end of the file is reached immediately. +.. c:function:: int PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction handler) + + Overrides the normal behavior of :func:`io.open_code` to pass its parameter + through the provided handler. + + The handler is a function of type :c:type:`PyObject *(\*)(PyObject *path, + void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`. + + The *userData* pointer is passed into the hook function. Since hook + functions may be called from different runtimes, this pointer should not + refer directly to Python state. + + As this hook is intentionally used during import, avoid importing new modules + during its execution unless they are known to be frozen or available in + ``sys.modules``. + + Once a hook has been set, it cannot be removed or replaced, and later calls to + :c:func:`PyFile_SetOpenCodeHook` will fail. On failure, the function returns + -1 and sets an exception if the interpreter has been initialized. + + This function is safe to call before :c:func:`Py_Initialize`. + + .. versionadded:: 3.8 + + + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) .. index:: single: Py_PRINT_RAW diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 8a996422ce48f8..057ff522516a3a 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -47,9 +47,13 @@ Floating Point Objects Return a C :c:type:`double` representation of the contents of *pyfloat*. If *pyfloat* is not a Python floating point object but has a :meth:`__float__` method, this method will first be called to convert *pyfloat* into a float. + If ``__float__()`` is not defined then it falls back to :meth:`__index__`. This method returns ``-1.0`` upon failure, so one should call :c:func:`PyErr_Occurred` to check for errors. + .. versionchanged:: 3.8 + Use :meth:`__index__` if available. + .. c:function:: double PyFloat_AS_DOUBLE(PyObject *pyfloat) diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index 3bfbaf4eedde0e..9a8f1507b3f4cc 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -21,6 +21,7 @@ document the API functions in detail. abstract.rst concrete.rst init.rst + init_config.rst memory.rst objimpl.rst apiabiversion.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst new file mode 100644 index 00000000000000..0d94e6b8f27b57 --- /dev/null +++ b/Doc/c-api/init_config.rst @@ -0,0 +1,1018 @@ +.. highlight:: c + +.. _init-config: + +*********************************** +Python Initialization Configuration +*********************************** + +.. versionadded:: 3.8 + +Structures: + +* :c:type:`PyConfig` +* :c:type:`PyPreConfig` +* :c:type:`PyStatus` +* :c:type:`PyWideStringList` + +Functions: + +* :c:func:`PyConfig_Clear` +* :c:func:`PyConfig_InitIsolatedConfig` +* :c:func:`PyConfig_InitPythonConfig` +* :c:func:`PyConfig_Read` +* :c:func:`PyConfig_SetArgv` +* :c:func:`PyConfig_SetBytesArgv` +* :c:func:`PyConfig_SetBytesString` +* :c:func:`PyConfig_SetString` +* :c:func:`PyPreConfig_InitIsolatedConfig` +* :c:func:`PyPreConfig_InitPythonConfig` +* :c:func:`PyStatus_Error` +* :c:func:`PyStatus_Exception` +* :c:func:`PyStatus_Exit` +* :c:func:`PyStatus_IsError` +* :c:func:`PyStatus_IsExit` +* :c:func:`PyStatus_NoMemory` +* :c:func:`PyStatus_Ok` +* :c:func:`PyWideStringList_Append` +* :c:func:`PyWideStringList_Insert` +* :c:func:`Py_ExitStatusException` +* :c:func:`Py_InitializeFromConfig` +* :c:func:`Py_PreInitialize` +* :c:func:`Py_PreInitializeFromArgs` +* :c:func:`Py_PreInitializeFromBytesArgs` +* :c:func:`Py_RunMain` + +The preconfiguration (``PyPreConfig`` type) is stored in +``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in +``PyInterpreterState.config``. + +.. seealso:: + :pep:`587` "Python Initialization Configuration". + + +PyWideStringList +---------------- + +.. c:type:: PyWideStringList + + List of ``wchar_t*`` strings. + + If *length* is non-zero, *items* must be non-NULL and all strings must be + non-NULL. + + Methods: + + .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) + + Append *item* to *list*. + + Python must be preinitialized to call this function. + + .. c:function:: PyStatus PyWideStringList_Insert(PyWideStringList *list, Py_ssize_t index, const wchar_t *item) + + Insert *item* into *list* at *index*. If *index* is greater than *list* + length, just append *item* to *list*. + + Python must be preinitialized to call this function. + + Structure fields: + + .. c:member:: Py_ssize_t length + + List length. + + .. c:member:: wchar_t** items + + List items. + +PyStatus +-------- + +.. c:type:: PyStatus + + Structure to store an initialization function status: success, error + or exit. + + For an error, it can store the C function name which created the error. + + Structure fields: + + .. c:member:: int exitcode + + Exit code. Argument passed to ``exit()``. + + .. c:member:: const char *err_msg + + Error message. + + .. c:member:: const char *func + + Name of the function which created an error, can be ``NULL``. + + Functions to create a status: + + .. c:function:: PyStatus PyStatus_Ok(void) + + Success. + + .. c:function:: PyStatus PyStatus_Error(const char *err_msg) + + Initialization error with a message. + + .. c:function:: PyStatus PyStatus_NoMemory(void) + + Memory allocation failure (out of memory). + + .. c:function:: PyStatus PyStatus_Exit(int exitcode) + + Exit Python with the specified exit code. + + Functions to handle a status: + + .. c:function:: int PyStatus_Exception(PyStatus status) + + Is the status an error or an exit? If true, the exception must be + handled; by calling :c:func:`Py_ExitStatusException` for example. + + .. c:function:: int PyStatus_IsError(PyStatus status) + + Is the result an error? + + .. c:function:: int PyStatus_IsExit(PyStatus status) + + Is the result an exit? + + .. c:function:: void Py_ExitStatusException(PyStatus status) + + Call ``exit(exitcode)`` if *status* is an exit. Print the error + message and exit with a non-zero exit code if *status* is an error. Must + only be called if ``PyStatus_Exception(status)`` is non-zero. + +.. note:: + Internally, Python uses macros which set ``PyStatus.func``, + whereas functions to create a status set ``func`` to ``NULL``. + +Example:: + + PyStatus alloc(void **ptr, size_t size) + { + *ptr = PyMem_RawMalloc(size); + if (*ptr == NULL) { + return PyStatus_NoMemory(); + } + return PyStatus_Ok(); + } + + int main(int argc, char **argv) + { + void *ptr; + PyStatus status = alloc(&ptr, 16); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + PyMem_Free(ptr); + return 0; + } + + +PyPreConfig +----------- + +.. c:type:: PyPreConfig + + Structure used to preinitialize Python: + + * Set the Python memory allocator + * Configure the LC_CTYPE locale + * Set the UTF-8 mode + + Function to initialize a preconfiguration: + + .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + + Initialize the preconfiguration with :ref:`Python Configuration + `. + + .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) + + Initialize the preconfiguration with :ref:`Isolated Configuration + `. + + Structure fields: + + .. c:member:: int allocator + + Name of the memory allocator: + + * ``PYMEM_ALLOCATOR_NOT_SET`` (``0``): don't change memory allocators + (use defaults) + * ``PYMEM_ALLOCATOR_DEFAULT`` (``1``): default memory allocators + * ``PYMEM_ALLOCATOR_DEBUG`` (``2``): default memory allocators with + debug hooks + * ``PYMEM_ALLOCATOR_MALLOC`` (``3``): force usage of ``malloc()`` + * ``PYMEM_ALLOCATOR_MALLOC_DEBUG`` (``4``): force usage of + ``malloc()`` with debug hooks + * ``PYMEM_ALLOCATOR_PYMALLOC`` (``5``): :ref:`Python pymalloc memory + allocator ` + * ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc + memory allocator ` with debug hooks + + ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` + are not supported if Python is configured using ``--without-pymalloc`` + + See :ref:`Memory Management `. + + .. c:member:: int configure_locale + + Set the LC_CTYPE locale to the user preferred locale? If equals to 0, set + :c:member:`coerce_c_locale` and :c:member:`coerce_c_locale_warn` to 0. + + .. c:member:: int coerce_c_locale + + If equals to 2, coerce the C locale; if equals to 1, read the LC_CTYPE + locale to decide if it should be coerced. + + .. c:member:: int coerce_c_locale_warn + If non-zero, emit a warning if the C locale is coerced. + + .. c:member:: int dev_mode + + See :c:member:`PyConfig.dev_mode`. + + .. c:member:: int isolated + + See :c:member:`PyConfig.isolated`. + + .. c:member:: int legacy_windows_fs_encoding (Windows only) + + If non-zero, disable UTF-8 Mode, set the Python filesystem encoding to + ``mbcs``, set the filesystem error handler to ``replace``. + + Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for + Windows specific code. + + .. c:member:: int parse_argv + + If non-zero, :c:func:`Py_PreInitializeFromArgs` and + :c:func:`Py_PreInitializeFromBytesArgs` parse their ``argv`` argument the + same way the regular Python parses command line arguments: see + :ref:`Command Line Arguments `. + + .. c:member:: int use_environment + + See :c:member:`PyConfig.use_environment`. + + .. c:member:: int utf8_mode + + If non-zero, enable the UTF-8 mode. + +Preinitialization with PyPreConfig +---------------------------------- + +Functions to preinitialize Python: + +.. c:function:: PyStatus Py_PreInitialize(const PyPreConfig *preconfig) + + Preinitialize Python from *preconfig* preconfiguration. + +.. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) + + Preinitialize Python from *preconfig* preconfiguration and command line + arguments (bytes strings). + +.. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) + + Preinitialize Python from *preconfig* preconfiguration and command line + arguments (wide strings). + +The caller is responsible to handle exceptions (error or exit) using +:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. + +For :ref:`Python Configuration ` +(:c:func:`PyPreConfig_InitPythonConfig`), if Python is initialized with +command line arguments, the command line arguments must also be passed to +preinitialize Python, since they have an effect on the pre-configuration +like encodings. For example, the :option:`-X` ``utf8`` command line option +enables the UTF-8 Mode. + +``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and +before :c:func:`Py_InitializeFromConfig` to install a custom memory allocator. +It can be called before :c:func:`Py_PreInitialize` if +:c:member:`PyPreConfig.allocator` is set to ``PYMEM_ALLOCATOR_NOT_SET``. + +Python memory allocation functions like :c:func:`PyMem_RawMalloc` must not be +used before Python preinitialization, whereas calling directly ``malloc()`` and +``free()`` is always safe. :c:func:`Py_DecodeLocale` must not be called before +the preinitialization. + +Example using the preinitialization to enable the UTF-8 Mode:: + + PyPreConfig preconfig; + PyPreConfig_InitPythonConfig(&preconfig); + + preconfig.utf8_mode = 1; + + PyStatus status = Py_PreInitialize(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + /* at this point, Python will speak UTF-8 */ + + Py_Initialize(); + /* ... use Python API here ... */ + Py_Finalize(); + + +PyConfig +-------- + +.. c:type:: PyConfig + + Structure containing most parameters to configure Python. + + Structure methods: + + .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config) + + Initialize configuration with :ref:`Python Configuration + `. + + .. c:function:: PyStatus PyConfig_InitIsolatedConfig(PyConfig *config) + + Initialize configuration with :ref:`Isolated Configuration + `. + + .. c:function:: PyStatus PyConfig_SetString(PyConfig *config, wchar_t * const *config_str, const wchar_t *str) + + Copy the wide character string *str* into ``*config_str``. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetBytesString(PyConfig *config, wchar_t * const *config_str, const char *str) + + Decode *str* using ``Py_DecodeLocale()`` and set the result into ``*config_str``. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetArgv(PyConfig *config, int argc, wchar_t * const *argv) + + Set command line arguments from wide character strings. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_SetBytesArgv(PyConfig *config, int argc, char * const *argv) + + Set command line arguments: decode bytes using :c:func:`Py_DecodeLocale`. + + Preinitialize Python if needed. + + .. c:function:: PyStatus PyConfig_Read(PyConfig *config) + + Read all Python configuration. + + Fields which are already initialized are left unchanged. + + Preinitialize Python if needed. + + .. c:function:: void PyConfig_Clear(PyConfig *config) + + Release configuration memory. + + Most ``PyConfig`` methods preinitialize Python if needed. In that case, the + Python preinitialization configuration in based on the :c:type:`PyConfig`. + If configuration fields which are in common with :c:type:`PyPreConfig` are + tuned, they must be set before calling a :c:type:`PyConfig` method: + + * :c:member:`~PyConfig.dev_mode` + * :c:member:`~PyConfig.isolated` + * :c:member:`~PyConfig.parse_argv` + * :c:member:`~PyConfig.use_environment` + + Moreover, if :c:func:`PyConfig_SetArgv` or :c:func:`PyConfig_SetBytesArgv` + is used, this method must be called first, before other methods, since the + preinitialization configuration depends on command line arguments (if + :c:member:`parse_argv` is non-zero). + + The caller of these methods is responsible to handle exceptions (error or + exit) using ``PyStatus_Exception()`` and ``Py_ExitStatusException()``. + + Structure fields: + + .. c:member:: PyWideStringList argv + + Command line arguments, :data:`sys.argv`. See + :c:member:`~PyConfig.parse_argv` to parse :c:member:`~PyConfig.argv` the + same way the regular Python parses Python command line arguments. If + :c:member:`~PyConfig.argv` is empty, an empty string is added to ensure + that :data:`sys.argv` always exists and is never empty. + + .. c:member:: wchar_t* base_exec_prefix + + :data:`sys.base_exec_prefix`. + + .. c:member:: wchar_t* base_prefix + + :data:`sys.base_prefix`. + + .. c:member:: int buffered_stdio + + If equals to 0, enable unbuffered mode, making the stdout and stderr + streams unbuffered. + + stdin is always opened in buffered mode. + + .. c:member:: int bytes_warning + + If equals to 1, issue a warning when comparing :class:`bytes` or + :class:`bytearray` with :class:`str`, or comparing :class:`bytes` with + :class:`int`. If equal or greater to 2, raise a :exc:`BytesWarning` + exception. + + .. c:member:: wchar_t* check_hash_pycs_mode + + Control the validation behavior of hash-based ``.pyc`` files (see + :pep:`552`): :option:`--check-hash-based-pycs` command line option value. + + Valid values: ``always``, ``never`` and ``default``. + + The default value is: ``default``. + + .. c:member:: int configure_c_stdio + + If non-zero, configure C standard streams (``stdio``, ``stdout``, + ``stdout``). For example, set their mode to ``O_BINARY`` on Windows. + + .. c:member:: int dev_mode + + Development mode: see :option:`-X` ``dev``. + + .. c:member:: int dump_refs + + If non-zero, dump all objects which are still alive at exit. + + Require a debug build of Python (``Py_REF_DEBUG`` macro must be defined). + + .. c:member:: wchar_t* exec_prefix + + :data:`sys.exec_prefix`. + + .. c:member:: wchar_t* executable + + :data:`sys.executable`. + + .. c:member:: int faulthandler + + If non-zero, call :func:`faulthandler.enable`. + + .. c:member:: wchar_t* filesystem_encoding + + Filesystem encoding, :func:`sys.getfilesystemencoding`. + + .. c:member:: wchar_t* filesystem_errors + + Filesystem encoding errors, :func:`sys.getfilesystemencodeerrors`. + + .. c:member:: unsigned long hash_seed + .. c:member:: int use_hash_seed + + Randomized hash function seed. + + If :c:member:`~PyConfig.use_hash_seed` is zero, a seed is chosen randomly + at Pythonstartup, and :c:member:`~PyConfig.hash_seed` is ignored. + + .. c:member:: wchar_t* home + + Python home directory. + + .. c:member:: int import_time + + If non-zero, profile import time. + + .. c:member:: int inspect + + Enter interactive mode after executing a script or a command. + + .. c:member:: int install_signal_handlers + + Install signal handlers? + + .. c:member:: int interactive + + Interactive mode. + + .. c:member:: int isolated + + If greater than 0, enable isolated mode: + + * :data:`sys.path` contains neither the script's directory (computed from + ``argv[0]`` or the current directory) nor the user's site-packages + directory. + * Python REPL doesn't import :mod:`readline` nor enable default readline + configuration on interactive prompts. + * Set :c:member:`~PyConfig.use_environment` and + :c:member:`~PyConfig.user_site_directory` to 0. + + .. c:member:: int legacy_windows_stdio + + If non-zero, use :class:`io.FileIO` instead of + :class:`io.WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout` + and :data:`sys.stderr`. + + Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for + Windows specific code. + + .. c:member:: int malloc_stats + + If non-zero, dump statistics on :ref:`Python pymalloc memory allocator + ` at exit. + + The option is ignored if Python is built using ``--without-pymalloc``. + + .. c:member:: wchar_t* pythonpath_env + + Module search paths as a string separated by ``DELIM`` + (:data:`os.path.pathsep`). + + Initialized from :envvar:`PYTHONPATH` environment variable value by + default. + + .. c:member:: PyWideStringList module_search_paths + .. c:member:: int module_search_paths_set + + :data:`sys.path`. If :c:member:`~PyConfig.module_search_paths_set` is + equal to 0, the :c:member:`~PyConfig.module_search_paths` is overridden + by the function computing the :ref:`Path Configuration + `. + + .. c:member:: int optimization_level + + Compilation optimization level: + + * 0: Peephole optimizer (and ``__debug__`` is set to ``True``) + * 1: Remove assertions, set ``__debug__`` to ``False`` + * 2: Strip docstrings + + .. c:member:: int parse_argv + + If non-zero, parse :c:member:`~PyConfig.argv` the same way the regular + Python command line arguments, and strip Python arguments from + :c:member:`~PyConfig.argv`: see :ref:`Command Line Arguments + `. + + .. c:member:: int parser_debug + + If non-zero, turn on parser debugging output (for expert only, depending + on compilation options). + + .. c:member:: int pathconfig_warnings + + If equal to 0, suppress warnings when computing the path configuration + (Unix only, Windows does not log any warning). Otherwise, warnings are + written into ``stderr``. + + .. c:member:: wchar_t* prefix + + :data:`sys.prefix`. + + .. c:member:: wchar_t* program_name + + Program name. + + .. c:member:: wchar_t* pycache_prefix + + ``.pyc`` cache prefix. + + .. c:member:: int quiet + + Quiet mode. For example, don't display the copyright and version messages + even in interactive mode. + + .. c:member:: wchar_t* run_command + + ``python3 -c COMMAND`` argument. + + .. c:member:: wchar_t* run_filename + + ``python3 FILENAME`` argument. + + .. c:member:: wchar_t* run_module + + ``python3 -m MODULE`` argument. + + .. c:member:: int show_alloc_count + + Show allocation counts at exit? + + Need a special Python build with ``COUNT_ALLOCS`` macro defined. + + .. c:member:: int show_ref_count + + Show total reference count at exit? + + Need a debug build of Python (``Py_REF_DEBUG`` macro must be defined). + + .. c:member:: int site_import + + Import the :mod:`site` module at startup? + + .. c:member:: int skip_source_first_line + + Skip the first line of the source? + + .. c:member:: wchar_t* stdio_encoding + .. c:member:: wchar_t* stdio_errors + + Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and + :data:`sys.stderr`. + + .. c:member:: int tracemalloc + + If non-zero, call :func:`tracemalloc.start`. + + .. c:member:: int use_environment + + If greater than 0, use :ref:`environment variables `. + + .. c:member:: int user_site_directory + + If non-zero, add user site directory to :data:`sys.path`. + + .. c:member:: int verbose + + If non-zero, enable verbose mode. + + .. c:member:: PyWideStringList warnoptions + + Options of the :mod:`warnings` module to build warnings filters. + + .. c:member:: int write_bytecode + + If non-zero, write ``.pyc`` files. + + .. c:member:: PyWideStringList xoptions + + :data:`sys._xoptions`. + +If ``parse_argv`` is non-zero, ``argv`` arguments are parsed the same +way the regular Python parses command line arguments, and Python +arguments are stripped from ``argv``: see :ref:`Command Line Arguments +`. + +The ``xoptions`` options are parsed to set other options: see :option:`-X` +option. + + +Initialization with PyConfig +---------------------------- + +Function to initialize Python: + +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration. + +The caller is responsible to handle exceptions (error or exit) using +:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. + +``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or +``PyImport_ExtendInittab()`` is used: they must be set or called after Python +preinitialization and before the Python initialization. + +Example setting the program name:: + + void init_python(void) + { + PyStatus status; + PyConfig config; + + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + + /* Set the program name. Implicitly preinitialize Python. */ + status = PyConfig_SetString(&config, &config.program_name, + L"/path/to/my_program"); + if (PyStatus_Exception(status)) { + goto fail; + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + PyConfig_Clear(&config); + return; + + fail: + PyConfig_Clear(&config); + Py_ExitStatusException(status); + } + +More complete example modifying the default configuration, read the +configuration, and then override some parameters:: + + PyStatus init_python(const char *program_name) + { + PyStatus status; + PyConfig config; + + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Set the program name before reading the configuraton + (decode byte string from the locale encoding). + + Implicitly preinitialize Python. */ + status = PyConfig_SetBytesString(&config, &config.program_name, + program_name); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Read all configuration at once */ + status = PyConfig_Read(&config); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Append our custom search path to sys.path */ + status = PyWideStringList_Append(&config.module_search_paths, + L"/path/to/more/modules"); + if (PyStatus_Exception(status)) { + goto done; + } + + /* Override executable computed by PyConfig_Read() */ + status = PyConfig_SetString(&config, &config.executable, + L"/path/to/my_executable"); + if (PyStatus_Exception(status)) { + goto done; + } + + status = Py_InitializeFromConfig(&config); + + done: + PyConfig_Clear(&config); + return status; + } + + +.. _init-isolated-conf: + +Isolated Configuration +---------------------- + +:c:func:`PyPreConfig_InitIsolatedConfig` and +:c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to +isolate Python from the system. For example, to embed Python into an +application. + +This configuration ignores global configuration variables, environments +variables and command line arguments (:c:member:`PyConfig.argv` is not parsed). +The C standard streams (ex: ``stdout``) and the LC_CTYPE locale are left +unchanged by default. + +Configuration files are still used with this configuration. Set the +:ref:`Path Configuration ` ("output fields") to ignore these +configuration files and avoid the function computing the default path +configuration. + + +.. _init-python-config: + +Python Configuration +-------------------- + +:c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` +functions create a configuration to build a customized Python which behaves as +the regular Python. + +Environments variables and command line arguments are used to configure +Python, whereas global configuration variables are ignored. + +This function enables C locale coercion (:pep:`538`) and UTF-8 Mode +(:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and +:envvar:`PYTHONCOERCECLOCALE` environment variables. + +Example of customized Python always running in isolated mode:: + + int main(int argc, char **argv) + { + PyConfig config; + PyStatus status; + + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + + config.isolated = 1; + + /* Decode command line arguments. + Implicitly preinitialize Python (in isolated mode). */ + status = PyConfig_SetBytesArgv(&config, argc, argv); + if (PyStatus_Exception(status)) { + goto fail; + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + PyConfig_Clear(&config); + + return Py_RunMain(); + + fail: + PyConfig_Clear(&config); + if (PyStatus_IsExit(status)) { + return status.exitcode; + } + /* Display the error message and exit the process with + non-zero exit code */ + Py_ExitStatusException(status); + } + + +.. _init-path-config: + +Path Configuration +------------------ + +:c:type:`PyConfig` contains multiple fields for the path configuration: + +* Path configuration input fields: + + * :c:member:`PyConfig.home` + * :c:member:`PyConfig.pythonpath_env` + * :c:member:`PyConfig.pathconfig_warnings` + +* Path configuration output fields: + + * :c:member:`PyConfig.exec_prefix` + * :c:member:`PyConfig.executable` + * :c:member:`PyConfig.prefix` + * :c:member:`PyConfig.module_search_paths_set`, + :c:member:`PyConfig.module_search_paths` + +If at least one "output field" is not set, Python computes the path +configuration to fill unset fields. If +:c:member:`~PyConfig.module_search_paths_set` is equal to 0, +:c:member:`~PyConfig.module_search_paths` is overriden and +:c:member:`~PyConfig.module_search_paths_set` is set to 1. + +It is possible to completely ignore the function computing the default +path configuration by setting explicitly all path configuration output +fields listed above. A string is considered as set even if it is non-empty. +``module_search_paths`` is considered as set if +``module_search_paths_set`` is set to 1. In this case, path +configuration input fields are ignored as well. + +Set :c:member:`~PyConfig.pathconfig_warnings` to 0 to suppress warnings when +computing the path configuration (Unix only, Windows does not log any warning). + +If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix` +fields are not set, they inherit their value from :c:member:`~PyConfig.prefix` +and :c:member:`~PyConfig.exec_prefix` respectively. + +:c:func:`Py_RunMain` and :c:func:`Py_Main` modify :data:`sys.path`: + +* If :c:member:`~PyConfig.run_filename` is set and is a directory which contains a + ``__main__.py`` script, prepend :c:member:`~PyConfig.run_filename` to + :data:`sys.path`. +* If :c:member:`~PyConfig.isolated` is zero: + + * If :c:member:`~PyConfig.run_module` is set, prepend the current directory + to :data:`sys.path`. Do nothing if the current directory cannot be read. + * If :c:member:`~PyConfig.run_filename` is set, prepend the directory of the + filename to :data:`sys.path`. + * Otherwise, prepend an empty string to :data:`sys.path`. + +If :c:member:`~PyConfig.site_import` is non-zero, :data:`sys.path` can be +modified by the :mod:`site` module. If +:c:member:`~PyConfig.user_site_directory` is non-zero and the user's +site-package directory exists, the :mod:`site` module appends the user's +site-package directory to :data:`sys.path`. + +The following configuration files are used by the path configuration: + +* ``pyvenv.cfg`` +* ``python._pth`` (Windows only) +* ``pybuilddir.txt`` (Unix only) + + +Py_RunMain() +------------ + +.. c:function:: int Py_RunMain(void) + + Execute the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. + + By default and when if :option:`-i` option is used, run the REPL. + + Finally, finalizes Python and returns an exit status that can be passed to + the ``exit()`` function. + +See :ref:`Python Configuration ` for an example of +customized Python always running in isolated mode using +:c:func:`Py_RunMain`. + + +Multi-Phase Initialization Private Provisional API +-------------------------------------------------- + +This section is a private provisional API introducing multi-phase +initialization, the core feature of the :pep:`432`: + +* "Core" initialization phase, "bare minimum Python": + + * Builtin types; + * Builtin exceptions; + * Builtin and frozen modules; + * The :mod:`sys` module is only partially initialized + (ex: :data:`sys.path` doesn't exist yet); + +* "Main" initialization phase, Python is fully initialized: + + * Install and configure :mod:`importlib`; + * Apply the :ref:`Path Configuration `; + * Install signal handlers; + * Finish :mod:`sys` module initialization (ex: create :data:`sys.stdout` + and :data:`sys.path`); + * Enable optional features like :mod:`faulthandler` and :mod:`tracemalloc`; + * Import the :mod:`site` module; + * etc. + +Private provisional API: + +* :c:member:`PyConfig._init_main`: if set to 0, + :c:func:`Py_InitializeFromConfig` stops at the "Core" initialization phase. + +.. c:function:: PyStatus _Py_InitializeMain(void) + + Move to the "Main" initialization phase, finish the Python initialization. + +No module is imported during the "Core" phase and the ``importlib`` module is +not configured: the :ref:`Path Configuration ` is only +applied during the "Main" phase. It may allow to customize Python in Python to +override or tune the :ref:`Path Configuration `, maybe +install a custom sys.meta_path importer or an import hook, etc. + +It may become possible to compute the :ref:`Path Configuration +` in Python, after the Core phase and before the Main phase, +which is one of the :pep:`432` motivation. + +The "Core" phase is not properly defined: what should be and what should +not be available at this phase is not specified yet. The API is marked +as private and provisional: the API can be modified or even be removed +anytime until a proper public API is designed. + +Example running Python code between "Core" and "Main" initialization +phases:: + + void init_python(void) + { + PyStatus status; + PyConfig config; + + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + PyConfig_Clear(&config); + Py_ExitStatusException(status); + } + + config._init_main = 0; + + /* ... customize 'config' configuration ... */ + + status = Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + /* Use sys.stderr because sys.stdout is only created + by _Py_InitializeMain() */ + int res = PyRun_SimpleString( + "import sys; " + "print('Run Python code before _Py_InitializeMain', " + "file=sys.stderr)"); + if (res < 0) { + exit(1); + } + + /* ... put more configuration code here ... */ + + status = _Py_InitializeMain(); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + } diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index b003bbaeff2f5b..a1c8d34a7ea02f 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -156,10 +156,22 @@ complete listing. .. c:macro:: Py_UNUSED(arg) Use this for unused arguments in a function definition to silence compiler - warnings, e.g. ``PyObject* func(PyObject *Py_UNUSED(ignored))``. + warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. .. versionadded:: 3.4 +.. c:macro:: Py_DEPRECATED(version) + + Use this for deprecated declarations. The macro must be placed before the + symbol name. + + Example:: + + Py_DEPRECATED(3.8) PyAPI_FUNC(int) Py_OldFunction(void); + + .. versionchanged:: 3.8 + MSVC support was added. + .. _api-objects: diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 6be29f9cd32eaa..fdaefafe21ba09 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -288,7 +288,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long`, return the reduction of that value modulo ``ULONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long)-1`` on error. Use :c:func:`PyErr_Occurred` to + disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. @@ -307,7 +308,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long long`, return the reduction of that value modulo ``PY_ULLONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long long)-1`` on error. Use :c:func:`PyErr_Occurred` + to disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index ffc35241e7a4d7..ce0d05942f4e79 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -335,6 +335,83 @@ Object Protocol *NULL* on failure. +.. c:function:: PyObject* _PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + Call a callable Python object *callable*, using + :c:data:`vectorcall ` if possible. + + *args* is a C array with the positional arguments. + + *nargsf* is the number of positional arguments plus optionally the flag + :const:`PY_VECTORCALL_ARGUMENTS_OFFSET` (see below). + To get actual number of arguments, use + :c:func:`PyVectorcall_NARGS(nargsf) `. + + *kwnames* can be either NULL (no keyword arguments) or a tuple of keyword + names. In the latter case, the values of the keyword arguments are stored + in *args* after the positional arguments. + The number of keyword arguments does not influence *nargsf*. + + *kwnames* must contain only objects of type ``str`` (not a subclass), + and all keys must be unique. + + Return the result of the call on success, or *NULL* on failure. + + This uses the vectorcall protocol if the callable supports it; + otherwise, the arguments are converted to use + :c:member:`~PyTypeObject.tp_call`. + + .. note:: + + This function is provisional and expected to become public in Python 3.9, + with a different name and, possibly, changed semantics. + If you use the function, plan for updating your code for Python 3.9. + + .. versionadded:: 3.8 + +.. c:var:: PY_VECTORCALL_ARGUMENTS_OFFSET + + If set in a vectorcall *nargsf* argument, the callee is allowed to + temporarily change ``args[-1]``. In other words, *args* points to + argument 1 (not 0) in the allocated vector. + The callee must restore the value of ``args[-1]`` before returning. + + Whenever they can do so cheaply (without additional allocation), callers + are encouraged to use :const:`PY_VECTORCALL_ARGUMENTS_OFFSET`. + Doing so will allow callables such as bound methods to make their onward + calls (which include a prepended *self* argument) cheaply. + + .. versionadded:: 3.8 + +.. c:function:: Py_ssize_t PyVectorcall_NARGS(size_t nargsf) + + Given a vectorcall *nargsf* argument, return the actual number of + arguments. + Currently equivalent to ``nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET``. + + .. versionadded:: 3.8 + +.. c:function:: PyObject* _PyObject_FastCallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) + + Same as :c:func:`_PyObject_Vectorcall` except that the keyword arguments + are passed as a dictionary in *kwdict*. This may be *NULL* if there + are no keyword arguments. + + For callables supporting :c:data:`vectorcall `, + the arguments are internally converted to the vectorcall convention. + Therefore, this function adds some overhead compared to + :c:func:`_PyObject_Vectorcall`. + It should only be used if the caller already has a dictionary ready to use. + + .. note:: + + This function is provisional and expected to become public in Python 3.9, + with a different name and, possibly, changed semantics. + If you use the function, plan for updating your code for Python 3.9. + + .. versionadded:: 3.8 + + .. c:function:: Py_hash_t PyObject_Hash(PyObject *o) .. index:: builtin: hash diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 5e0cfd0264f931..5184ad511cd9b1 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -114,10 +114,20 @@ the definition of all other Python objects. .. c:type:: PyCFunctionWithKeywords - Type of the functions used to implement Python callables in C that take - keyword arguments: they take three :c:type:`PyObject\*` parameters and return - one such value. See :c:type:`PyCFunction` above for the meaning of the return - value. + Type of the functions used to implement Python callables in C + with signature :const:`METH_VARARGS | METH_KEYWORDS`. + + +.. c:type:: _PyCFunctionFast + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL`. + + +.. c:type:: _PyCFunctionFastWithKeywords + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL | METH_KEYWORDS`. .. c:type:: PyMethodDef @@ -149,10 +159,11 @@ specific C type of the *self* object. The :attr:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding -convention. Of the calling convention flags, only :const:`METH_VARARGS` and -:const:`METH_KEYWORDS` can be combined. Any of the calling convention flags -can be combined with a binding flag. +convention. +There are four basic calling conventions for positional arguments +and two of them can be combined with :const:`METH_KEYWORDS` to support +also keyword arguments. So there are a total of 6 calling conventions: .. data:: METH_VARARGS @@ -164,13 +175,41 @@ can be combined with a binding flag. using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`. -.. data:: METH_KEYWORDS +.. data:: METH_VARARGS | METH_KEYWORDS Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`. - The function expects three parameters: *self*, *args*, and a dictionary of - all the keyword arguments. The flag must be combined with - :const:`METH_VARARGS`, and the parameters are typically processed using - :c:func:`PyArg_ParseTupleAndKeywords`. + The function expects three parameters: *self*, *args*, *kwargs* where + *kwargs* is a dictionary of all the keyword arguments or possibly *NULL* + if there are no keyword arguments. The parameters are typically processed + using :c:func:`PyArg_ParseTupleAndKeywords`. + + +.. data:: METH_FASTCALL + + Fast calling convention supporting only positional arguments. + The methods have the type :c:type:`_PyCFunctionFast`. + The first parameter is *self*, the second parameter is a C array + of :c:type:`PyObject\*` values indicating the arguments and the third + parameter is the number of arguments (the length of the array). + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 + + +.. data:: METH_FASTCALL | METH_KEYWORDS + + Extension of :const:`METH_FASTCALL` supporting also keyword arguments, + with methods of type :c:type:`_PyCFunctionFastWithKeywords`. + Keyword arguments are passed the same way as in the vectorcall protocol: + there is an additional fourth :c:type:`PyObject\*` parameter + which is a tuple representing the names of the keyword arguments + or possibly *NULL* if there are no keywords. The values of the keyword + arguments are stored in the *args* array, after the positional arguments. + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 .. data:: METH_NOARGS diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 04e169a00dc699..7d870a8d4e4a71 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -289,6 +289,56 @@ accessible to C code. They all work with the current interpreter thread's .. versionadded:: 3.2 +.. c:function:: int PySys_Audit(const char *event, const char *format, ...) + + .. index:: single: audit events + + Raises an auditing event with any active hooks. Returns zero for success + and non-zero with an exception set on failure. + + If any hooks have been added, *format* and other arguments will be used + to construct a tuple to pass. Apart from ``N``, the same format characters + as used in :c:func:`Py_BuildValue` are available. If the built value is not + a tuple, it will be added into a single-element tuple. (The ``N`` format + option consumes a reference, but since there is no way to know whether + arguments to this function will be consumed, using it may cause reference + leaks.) + + :func:`sys.audit` performs the same function from Python code. + + .. versionadded:: 3.8 + + +.. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) + + .. index:: single: audit events + + Adds to the collection of active auditing hooks. Returns zero for success + and non-zero on failure. If the runtime has been initialized, also sets an + error on failure. Hooks added through this API are called for all + interpreters created by the runtime. + + This function is safe to call before :c:func:`Py_Initialize`. When called + after runtime initialization, existing audit hooks are notified and may + silently abort the operation by raising an error subclassed from + :class:`Exception` (other errors will not be silenced). + + The hook function is of type :c:type:`int (*)(const char *event, PyObject + *args, void *userData)`, where *args* is guaranteed to be a + :c:type:`PyTupleObject`. The hook function is always called with the GIL + held by the Python interpreter that raised the event. + + The *userData* pointer is passed into the hook function. Since hook + functions may be called from different runtimes, this pointer should not + refer directly to Python state. + + See :pep:`578` for a detailed description of auditing. Functions in the + runtime and standard library that raise events include the details in each + function's documentation. + + .. versionadded:: 3.8 + + .. _processcontrol: Process Control diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 2474df2c90baad..8f8367ab77c8c4 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -95,18 +95,6 @@ Type Objects from a type's base class. Return ``0`` on success, or return ``-1`` and sets an exception on error. -.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) - - Creates and returns a heap type object from the *spec* passed to the function. - -.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) - - Creates and returns a heap type object from the *spec*. In addition to that, - the created heap type contains all types contained by the *bases* tuple as base - types. This allows the caller to reference other heap types as base types. - - .. versionadded:: 3.3 - .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -115,4 +103,107 @@ Type Objects Callers will typically cast the result pointer into the appropriate function type. + See :c:member:`PyType_Slot.slot` for possible values of the *slot* argument. + + An exception is raised if *type* is not a heap type. + .. versionadded:: 3.4 + + +Creating Heap-Allocated Types +............................. + +The following functions and structs are used to create +:ref:`heap types `. + +.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) + + Creates and returns a heap type object from the *spec*. + + If *bases* is a tuple, the created heap type contains all types contained + in it as base types. + + If *bases* is *NULL*, the *Py_tp_base* slot is used instead. + If that also is *NULL*, the new type derives from :class:`object`. + + This function calls :c:func:`PyType_Ready` on the new type. + + .. versionadded:: 3.3 + +.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) + + Equivalent to ``PyType_FromSpecWithBases(spec, NULL)``. + +.. c:type:: PyType_Spec + + Structure defining a type's behavior. + + .. c:member:: const char* PyType_Spec.name + + Name of the type, used to set :c:member:`PyTypeObject.tp_name`. + + .. c:member:: const char* PyType_Spec.doc + + Type docstring, used to set :c:member:`PyTypeObject.tp_doc`. + + .. c:member:: int PyType_Spec.basicsize + .. c:member:: int PyType_Spec.itemsize + + Size of the instance in bytes, used to set + :c:member:`PyTypeObject.tp_basicsize` and + :c:member:`PyTypeObject.tp_itemsize`. + + .. c:member:: int PyType_Spec.flags + + Type flags, used to set :c:member:`PyTypeObject.tp_flags`. + + If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, + :c:func:`PyType_FromSpecWithBases` sets it automatically. + + .. c:member:: PyType_Slot *PyType_Spec.slots + + Array of :c:type:`PyType_Slot` structures. + Terminated by the special slot value ``{0, NULL}``. + +.. c:type:: PyType_Slot + + Structure defining optional functionality of a type, containing a slot ID + and a value pointer. + + .. c:member:: int PyType_Slot.slot + + A slot ID. + + Slot IDs are named like the field names of the structures + :c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, + :c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and + :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. + For example, use: + + * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` + * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` + * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + + The following fields cannot be set using *PyType_Spec* and *PyType_Slot*: + + * :c:member:`~PyTypeObject.tp_dict` + * :c:member:`~PyTypeObject.tp_mro` + * :c:member:`~PyTypeObject.tp_cache` + * :c:member:`~PyTypeObject.tp_subclasses` + * :c:member:`~PyTypeObject.tp_weaklist` + * :c:member:`~PyTypeObject.tp_print` + * :c:member:`~PyTypeObject.tp_weaklistoffset` + * :c:member:`~PyTypeObject.tp_dictoffset` + * :c:member:`~PyBufferProcs.bf_getbuffer` + * :c:member:`~PyBufferProcs.bf_releasebuffer` + + Setting :c:data:`Py_tp_bases` may be problematic on some platforms. + To avoid issues, use the *bases* argument of + :py:func:`PyType_FromSpecWithBases` instead. + + .. c:member:: void *PyType_Slot.pfunc + + The desired value of the slot. In most cases, this is a pointer + to a function. + + May not be *NULL*. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index b1d96db7f53df0..83fcc5abed7079 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -36,115 +36,115 @@ Quick Reference .. table:: :widths: 18,18,18,1,1,1,1 - +---------------------------------------------+-----------------------------------+-------------------+---------------+ - | PyTypeObject Slot [#slots]_ | :ref:`Type ` | special | Info [#cols]_ | - | | | methods/attrs +---+---+---+---+ - | | | | O | T | D | I | - +=============================================+===================================+===================+===+===+===+===+ - | :c:member:`~PyTypeObject.tp_name` | const char * | __name__ | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_basicsize` | Py_ssize_t | | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_itemsize` | Py_ssize_t | | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dealloc` | :c:type:`destructor` | | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_print`) | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_getattr`) | :c:type:`getattrfunc` | __getattribute__, | | | | G | - | | | __getattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_setattr`) | :c:type:`setattrfunc` | __setattr__, | | | | G | - | | | __delattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_async` | :c:type:`PyAsyncMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_repr` | :c:type:`reprfunc` | __repr__ | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_number` | :c:type:`PyNumberMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_sequence` | :c:type:`PySequenceMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_mapping` | :c:type:`PyMappingMethods` * | :ref:`sub-slots` | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_hash` | :c:type:`hashfunc` | __hash__ | X | | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_call` | :c:type:`ternaryfunc` | __call__ | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_str` | :c:type:`reprfunc` | __str__ | X | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_getattro` | :c:type:`getattrofunc` | __getattribute__, | X | X | | G | - | | | __getattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | - | | | __delattr__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_doc` | const char * | __doc__ | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_traverse` | :c:type:`traverseproc` | | | X | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_clear` | :c:type:`inquiry` | | | X | | G | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_richcompare` | :c:type:`richcmpfunc` | __lt__, | X | | | G | - | | | __le__, | | | | | - | | | __eq__, | | | | | - | | | __ne__, | | | | | - | | | __gt__, | | | | | - | | | __ge__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_weaklistoffset` | Py_ssize_t | | | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_iter` | :c:type:`getiterfunc` | __iter__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_iternext` | :c:type:`iternextfunc` | __next__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_methods` | :c:type:`PyMethodDef` [] | | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_members` | :c:type:`PyMemberDef` [] | | | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_getset` | :c:type:`PyGetSetDef` [] | | X | X | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_base` | :c:type:`PyTypeObject` * | __base__ | | | X | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dict` | :c:type:`PyObject` * | __dict__ | | | ? | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_descr_get` | :c:type:`descrgetfunc` | __get__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_descr_set` | :c:type:`descrsetfunc` | __set__, | | | | X | - | | | __delete__ | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dictoffset` | Py_ssize_t | | | X | | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_init` | :c:type:`initproc` | __init__ | X | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_alloc` | :c:type:`allocfunc` | | X | | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_new` | :c:type:`newfunc` | __new__ | X | X | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_free` | :c:type:`freefunc` | | X | X | ? | ? | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_is_gc` | :c:type:`inquiry` | | | X | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | <:c:member:`~PyTypeObject.tp_bases`> | :c:type:`PyObject` * | __bases__ | | | ~ | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | <:c:member:`~PyTypeObject.tp_mro`> | :c:type:`PyObject` * | __mro__ | | | ~ | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_subclasses`] | :c:type:`PyObject` * | __subclasses__ | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_weaklist`] | :c:type:`PyObject` * | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_del`) | :c:type:`destructor` | | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_version_tag`] | unsigned int | | | | | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_finalize` | :c:type:`destructor` | __del__ | | | | X | - +---------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + +------------------------------------------------+-----------------------------------+-------------------+---------------+ + | PyTypeObject Slot [#slots]_ | :ref:`Type ` | special | Info [#cols]_ | + | | | methods/attrs +---+---+---+---+ + | | | | O | T | D | I | + +================================================+===================================+===================+===+===+===+===+ + | :c:member:`~PyTypeObject.tp_name` | const char * | __name__ | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_basicsize` | Py_ssize_t | | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_itemsize` | Py_ssize_t | | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dealloc` | :c:type:`destructor` | | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_vectorcall_offset` | Py_ssize_t | | | | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_getattr`) | :c:type:`getattrfunc` | __getattribute__, | | | | G | + | | | __getattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_setattr`) | :c:type:`setattrfunc` | __setattr__, | | | | G | + | | | __delattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_async` | :c:type:`PyAsyncMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_repr` | :c:type:`reprfunc` | __repr__ | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_number` | :c:type:`PyNumberMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_sequence` | :c:type:`PySequenceMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_mapping` | :c:type:`PyMappingMethods` * | :ref:`sub-slots` | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_hash` | :c:type:`hashfunc` | __hash__ | X | | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_call` | :c:type:`ternaryfunc` | __call__ | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_str` | :c:type:`reprfunc` | __str__ | X | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_getattro` | :c:type:`getattrofunc` | __getattribute__, | X | X | | G | + | | | __getattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | + | | | __delattr__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_doc` | const char * | __doc__ | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_traverse` | :c:type:`traverseproc` | | | X | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_clear` | :c:type:`inquiry` | | | X | | G | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_richcompare` | :c:type:`richcmpfunc` | __lt__, | X | | | G | + | | | __le__, | | | | | + | | | __eq__, | | | | | + | | | __ne__, | | | | | + | | | __gt__, | | | | | + | | | __ge__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_weaklistoffset` | Py_ssize_t | | | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_iter` | :c:type:`getiterfunc` | __iter__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_iternext` | :c:type:`iternextfunc` | __next__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_methods` | :c:type:`PyMethodDef` [] | | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_members` | :c:type:`PyMemberDef` [] | | | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_getset` | :c:type:`PyGetSetDef` [] | | X | X | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_base` | :c:type:`PyTypeObject` * | __base__ | | | X | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dict` | :c:type:`PyObject` * | __dict__ | | | ? | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_descr_get` | :c:type:`descrgetfunc` | __get__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_descr_set` | :c:type:`descrsetfunc` | __set__, | | | | X | + | | | __delete__ | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_dictoffset` | Py_ssize_t | | | X | | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_init` | :c:type:`initproc` | __init__ | X | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_alloc` | :c:type:`allocfunc` | | X | | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_new` | :c:type:`newfunc` | __new__ | X | X | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_free` | :c:type:`freefunc` | | X | X | ? | ? | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_is_gc` | :c:type:`inquiry` | | | X | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | <:c:member:`~PyTypeObject.tp_bases`> | :c:type:`PyObject` * | __bases__ | | | ~ | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | <:c:member:`~PyTypeObject.tp_mro`> | :c:type:`PyObject` * | __mro__ | | | ~ | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_subclasses`] | :c:type:`PyObject` * | __subclasses__ | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_weaklist`] | :c:type:`PyObject` * | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | (:c:member:`~PyTypeObject.tp_del`) | :c:type:`destructor` | | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | [:c:member:`~PyTypeObject.tp_version_tag`] | unsigned int | | | | | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ + | :c:member:`~PyTypeObject.tp_finalize` | :c:type:`destructor` | __del__ | | | | X | + +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ If :const:`COUNT_ALLOCS` is defined then the following (internal-only) fields exist as well: @@ -364,12 +364,6 @@ slot typedefs +-----------------------------+-----------------------------+----------------------+ | :c:type:`reprfunc` | :c:type:`PyObject` * | :c:type:`PyObject` * | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`printfunc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | FILE * | | -| | int | | -+-----------------------------+-----------------------------+----------------------+ | :c:type:`getattrfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | | | :c:type:`PyObject` * | | @@ -675,9 +669,66 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes. -.. c:member:: printfunc PyTypeObject.tp_print +.. c:member:: Py_ssize_t PyTypeObject.tp_vectorcall_offset + + An optional offset to a per-instance function that implements calling + the object using the *vectorcall* protocol, a more efficient alternative + of the simpler :c:member:`~PyTypeObject.tp_call`. + + This field is only used if the flag :const:`_Py_TPFLAGS_HAVE_VECTORCALL` + is set. If so, this must be a positive integer containing the offset in the + instance of a :c:type:`vectorcallfunc` pointer. + The signature is the same as for :c:func:`_PyObject_Vectorcall`:: + + PyObject *vectorcallfunc(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + The *vectorcallfunc* pointer may be zero, in which case the instance behaves + as if :const:`_Py_TPFLAGS_HAVE_VECTORCALL` was not set: calling the instance + falls back to :c:member:`~PyTypeObject.tp_call`. - Reserved slot, formerly used for print formatting in Python 2.x. + Any class that sets ``_Py_TPFLAGS_HAVE_VECTORCALL`` must also set + :c:member:`~PyTypeObject.tp_call` and make sure its behaviour is consistent + with the *vectorcallfunc* function. + This can be done by setting *tp_call* to ``PyVectorcall_Call``: + + .. c:function:: PyObject *PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict) + + Call *callable*'s *vectorcallfunc* with positional and keyword + arguments given in a tuple and dict, respectively. + + This function is intended to be used in the ``tp_call`` slot. + It does not fall back to ``tp_call`` and it currently does not check the + ``_Py_TPFLAGS_HAVE_VECTORCALL`` flag. + To call an object, use one of the :c:func:`PyObject_Call ` + functions instead. + + .. note:: + + It is not recommended for :ref:`heap types ` to implement + the vectorcall protocol. + When a user sets ``__call__`` in Python code, only ``tp_call`` is updated, + possibly making it inconsistent with the vectorcall function. + + .. note:: + + The semantics of the ``tp_vectorcall_offset`` slot are provisional and + expected to be finalized in Python 3.9. + If you use vectorcall, plan for updating your code for Python 3.9. + + .. versionchanged:: 3.8 + + This slot was used for print formatting in Python 2.x. + In Python 3.0 to 3.7, it was reserved and named ``tp_print``. + + **Inheritance:** + + This field is inherited by subtypes together with + :c:member:`~PyTypeObject.tp_call`: a subtype inherits + :c:member:`~PyTypeObject.tp_vectorcall_offset` from its base type when + the subtype’s :c:member:`~PyTypeObject.tp_call` is NULL. + + Note that `heap types`_ (including subclasses defined in Python) do not + inherit the :const:`_Py_TPFLAGS_HAVE_VECTORCALL` flag. .. c:member:: getattrfunc PyTypeObject.tp_getattr @@ -1045,6 +1096,32 @@ and :c:type:`PyType_Type` effectively act as defaults.) ??? + + .. data:: Py_TPFLAGS_METHOD_DESCRIPTOR + + This bit indicates that objects behave like unbound methods. + + If this flag is set for ``type(meth)``, then: + + - ``meth.__get__(obj, cls)(*args, **kwds)`` (with ``obj`` not None) + must be equivalent to ``meth(obj, *args, **kwds)``. + + - ``meth.__get__(None, cls)(*args, **kwds)`` + must be equivalent to ``meth(*args, **kwds)``. + + This flag enables an optimization for typical method calls like + ``obj.meth()``: it avoids creating a temporary "bound method" object for + ``obj.meth``. + + .. versionadded:: 3.8 + + **Inheritance:** + + This flag is never inherited by heap types. + For extension types, it is inherited whenever + :c:member:`~PyTypeObject.tp_descr_get` is inherited. + + .. XXX Document more flags here? @@ -1073,6 +1150,33 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.4 + .. deprecated:: 3.8 + This flag isn't necessary anymore, as the interpreter assumes the + :c:member:`~PyTypeObject.tp_finalize` slot is always present in the + type structure. + + .. data:: _Py_TPFLAGS_HAVE_VECTORCALL + + This bit is set when the class implements the vectorcall protocol. + See :c:member:`~PyTypeObject.tp_vectorcall_offset` for details. + + **Inheritance:** + + This bit is set on *static* subtypes if ``tp_flags`` is not overridden: + a subtype inherits ``_Py_TPFLAGS_HAVE_VECTORCALL`` from its base type + when the subtype’s :c:member:`~PyTypeObject.tp_call` is NULL + and the subtype's ``Py_TPFLAGS_HEAPTYPE`` is not set. + + `Heap types`_ do not inherit ``_Py_TPFLAGS_HAVE_VECTORCALL``. + + .. note:: + + This flag is provisional and expected to become public in Python 3.9, + with a different name and, possibly, changed semantics. + If you use vectorcall, plan for updating your code for Python 3.9. + + .. versionadded:: 3.8 + .. c:member:: const char* PyTypeObject.tp_doc @@ -1822,16 +1926,35 @@ objects on the thread which called tp_dealloc will not violate any assumptions of the library. +.. _heap-types: + Heap Types ---------- -In addition to defining Python types statically, you can define them -dynamically (i.e. to the heap) using :c:func:`PyType_FromSpec` and -:c:func:`PyType_FromSpecWithBases`. +Traditionally, types defined in C code are *static*, that is, +a static :c:type:`PyTypeObject` structure is defined directly in code +and initialized using :c:func:`PyType_Ready`. + +This results in types that are limited relative to types defined in Python: + +* Static types are limited to one base, i.e. they cannot use multiple + inheritance. +* Static type objects (but not necessarily their instances) are immutable. + It is not possible to add or modify the type object's attributes from Python. +* Static type objects are shared across + :ref:`sub-interpreters `, so they should not + include any subinterpreter-specific state. + +Also, since *PyTypeObject* is not part of the :ref:`stable ABI `, +any extension modules using static types must be compiled for a specific +Python minor version. -.. XXX Explain how to use PyType_FromSpec(). +An alternative to static types is *heap-allocated types*, or *heap types* +for short, which correspond closely to classes created by Python's +``class`` statement. -.. XXX Document PyType_Spec. +This is done by filling a :c:type:`PyType_Spec` structure and calling +:c:func:`PyType_FromSpecWithBases`. .. _number-structs: @@ -2236,6 +2359,14 @@ Slot Type typedefs .. c:type:: void (*destructor)(PyObject *) +.. c:type:: PyObject *(*vectorcallfunc)(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + + See :c:member:`~PyTypeObject.tp_vectorcall_offset`. + + Arguments to ``vectorcallfunc`` are the same as for :c:func:`_PyObject_Vectorcall`. + + .. versionadded:: 3.8 + .. c:type:: void (*freefunc)(void *) See :c:member:`~PyTypeObject.tp_free`. @@ -2252,10 +2383,6 @@ Slot Type typedefs See :c:member:`~PyTypeObject.tp_repr`. -.. c:type:: int (*printfunc)(PyObject *, FILE *, int) - - This is hidden if :const:`PY_LIMITED_API` is set. - .. c:type:: PyObject *(*getattrfunc)(PyObject *self, char *attr) Return the value of the named attribute for the object. @@ -2359,7 +2486,7 @@ with a more verbose initializer:: sizeof(MyObject), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)myobj_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index a7ff08cfb6bdaa..e6704ddeca0906 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -42,6 +42,13 @@ the same library that the Python runtime is using. ``Py_InspectFlag`` is not set. +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings. + + .. versionadded:: 3.8 + + .. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving @@ -381,11 +388,22 @@ the same library that the Python runtime is using. Whenever ``PyCompilerFlags *flags`` is *NULL*, :attr:`cf_flags` is treated as equal to ``0``, and any modification due to ``from __future__ import`` is - discarded. :: + discarded. + + .. c:member:: int cf_flags + + Compiler flags. + + .. c:member:: int cf_feature_version + + *cf_feature_version* is the minor Python version. It should be + initialized to ``PY_MINOR_VERSION``. + + The field is ignored by default, it is used if and only if + ``PyCF_ONLY_AST`` flag is set in *cf_flags*. - struct PyCompilerFlags { - int cf_flags; - } + .. versionchanged:: 3.8 + Added *cf_feature_version* field. .. c:var:: int CO_FUTURE_DIVISION diff --git a/Doc/conf.py b/Doc/conf.py index afe66270c10e8c..e85ea5b2d2ff49 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -23,6 +23,9 @@ except ImportError: _tkinter = None ''' + +manpages_url = 'https://manpages.debian.org/{path}' + # General substitutions. project = 'Python' copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y') diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 213ddcb61fae60..aca57a1dae9d8c 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -236,6 +236,7 @@ PyCode_GetNumFree:PyCodeObject*:co:0: PyCode_New:PyCodeObject*::+1: PyCode_New:int:argcount:: +PyCode_New:int:posonlyargcount:: PyCode_New:int:kwonlyargcount:: PyCode_New:int:nlocals:: PyCode_New:int:stacksize:: diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 5dd14b1f7a60b9..2e46c7ac8635e8 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -113,11 +113,17 @@ recommended tools`_. .. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations -Reading the guide -================= +.. index:: + single: Python Package Index (PyPI) + single: PyPI; (see Python Package Index (PyPI)) + +.. _publishing-python-packages: + +Reading the Python Packaging User Guide +======================================= The Python Packaging User Guide covers the various key steps and elements -involved in creating a project: +involved in creating and publishing a project: * `Project structure`_ * `Building and packaging the project`_ diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index cbeedab5bb1225..2601d30f63eb4c 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -290,7 +290,7 @@ the full reference. .. versionchanged:: 3.8 On Unix, C extensions are no longer linked to libpython except on - Android. + Android and Cygwin. .. class:: Distribution diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst index c56fafd68d8fdc..1f72a25542494a 100644 --- a/Doc/distutils/index.rst +++ b/Doc/distutils/index.rst @@ -36,7 +36,6 @@ and extensions readily available to a wider audience. configfile.rst sourcedist.rst builtdist.rst - packageindex.rst examples.rst extending.rst commandref.rst diff --git a/Doc/distutils/packageindex.rst b/Doc/distutils/packageindex.rst index f74c4396e54581..ccb9a598b2b7a2 100644 --- a/Doc/distutils/packageindex.rst +++ b/Doc/distutils/packageindex.rst @@ -1,6 +1,4 @@ -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) +:orphan: .. _package-index: @@ -12,6 +10,7 @@ The `Python Package Index (PyPI)`_ stores metadata describing distributions packaged with distutils and other publishing tools, as well the distribution archives themselves. -Detailed instructions on using PyPI at :ref:`distributing-index`. +References to up to date PyPI documentation can be found at +:ref:`publishing-python-packages`. .. _Python Package Index (PyPI): https://pypi.org diff --git a/Doc/distutils/uploading.rst b/Doc/distutils/uploading.rst index 4bce6997f9f83a..4c391cab072ea6 100644 --- a/Doc/distutils/uploading.rst +++ b/Doc/distutils/uploading.rst @@ -4,4 +4,5 @@ Uploading Packages to the Package Index *************************************** -The contents of this page have moved to the section :ref:`package-index`. +References to up to date PyPI documentation can be found at +:ref:`publishing-python-packages`. diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 4f9979bf55ed3a..781da467d18013 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -104,7 +104,7 @@ What platform-specific GUI toolkits exist for Python? ======================================================== By installing the `PyObjc Objective-C bridge -`_, Python programs can use Mac OS X's +`_, Python programs can use Mac OS X's Cocoa libraries. :ref:`Pythonwin ` by Mark Hammond includes an interface to the diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 9660a701427fd6..a00c6a053ef120 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -554,8 +554,8 @@ desired effect in a number of ways. 5) Or bundle up values in a class instance:: class callByRef: - def __init__(self, **args): - for (key, value) in args.items(): + def __init__(self, /, **args): + for key, value in args.items(): setattr(self, key, value) def func4(args): diff --git a/Doc/glossary.rst b/Doc/glossary.rst index d3ce3652551998..0f2a3a1fdf0510 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -225,7 +225,7 @@ Glossary statement by defining :meth:`__enter__` and :meth:`__exit__` methods. See :pep:`343`. - context variable + context variable A variable which can have different values depending on its context. This is similar to Thread-Local Storage in which each execution thread may have a different value for a variable. However, with context @@ -512,8 +512,10 @@ Glossary Hashability makes an object usable as a dictionary key and a set member, because these data structures use the hash value internally. - All of Python's immutable built-in objects are hashable; mutable - containers (such as lists or dictionaries) are not. Objects which are + Most of Python's immutable built-in objects are hashable; mutable + containers (such as lists or dictionaries) are not; immutable + containers (such as tuples and frozensets) are only hashable if + their elements are hashable. Objects which are instances of user-defined classes are hashable by default. They all compare unequal (except with themselves), and their hash value is derived from their :func:`id`. diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index b29e590b20cba8..324625d7b3e809 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -145,7 +145,7 @@ print a message for each get or set. Overriding :meth:`__getattribute__` is alternate approach that could do this for every attribute. However, this descriptor is useful for monitoring just a few chosen attributes:: - class RevealAccess(object): + class RevealAccess: """A data descriptor that sets and returns values normally and prints a message logging their access. """ @@ -162,7 +162,7 @@ descriptor is useful for monitoring just a few chosen attributes:: print('Updating', self.name) self.val = val - >>> class MyClass(object): + >>> class MyClass: ... x = RevealAccess(10, 'var "x"') ... y = 5 ... @@ -194,7 +194,7 @@ triggers function calls upon access to an attribute. Its signature is:: The documentation shows a typical use to define a managed attribute ``x``:: - class C(object): + class C: def getx(self): return self.__x def setx(self, value): self.__x = value def delx(self): del self.__x @@ -203,7 +203,7 @@ The documentation shows a typical use to define a managed attribute ``x``:: To see how :func:`property` is implemented in terms of the descriptor protocol, here is a pure Python equivalent:: - class Property(object): + class Property: "Emulate PyProperty_Type() in Objects/descrobject.c" def __init__(self, fget=None, fset=None, fdel=None, doc=None): @@ -250,7 +250,7 @@ to be recalculated on every access; however, the programmer does not want to affect existing client code accessing the attribute directly. The solution is to wrap access to the value attribute in a property data descriptor:: - class Cell(object): + class Cell: . . . def getvalue(self): "Recalculate the cell before returning value" @@ -277,7 +277,7 @@ binding methods during attribute access. This means that all functions are non-data descriptors which return bound methods when they are invoked from an object. In pure Python, it works like this:: - class Function(object): + class Function: . . . def __get__(self, obj, objtype=None): "Simulate func_descr_get() in Objects/funcobject.c" @@ -287,7 +287,7 @@ object. In pure Python, it works like this:: Running the interpreter shows how the function descriptor works in practice:: - >>> class D(object): + >>> class D: ... def f(self, x): ... return x ... @@ -367,7 +367,7 @@ It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` o Since staticmethods return the underlying function with no changes, the example calls are unexciting:: - >>> class E(object): + >>> class E: ... def f(x): ... print(x) ... f = staticmethod(f) @@ -380,7 +380,7 @@ calls are unexciting:: Using the non-data descriptor protocol, a pure Python version of :func:`staticmethod` would look like this:: - class StaticMethod(object): + class StaticMethod: "Emulate PyStaticMethod_Type() in Objects/funcobject.c" def __init__(self, f): @@ -393,7 +393,7 @@ Unlike static methods, class methods prepend the class reference to the argument list before calling the function. This format is the same for whether the caller is an object or a class:: - >>> class E(object): + >>> class E: ... def f(klass, x): ... return klass.__name__, x ... f = classmethod(f) @@ -410,7 +410,7 @@ is to create alternate class constructors. In Python 2.3, the classmethod :func:`dict.fromkeys` creates a new dictionary from a list of keys. The pure Python equivalent is:: - class Dict(object): + class Dict: . . . def fromkeys(klass, iterable, value=None): "Emulate dict_fromkeys() in Objects/dictobject.c" @@ -428,7 +428,7 @@ Now a new dictionary of unique keys can be constructed like this:: Using the non-data descriptor protocol, a pure Python version of :func:`classmethod` would look like this:: - class ClassMethod(object): + class ClassMethod: "Emulate PyClassMethod_Type() in Objects/funcobject.c" def __init__(self, f): diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 50cde3595034b5..909deb5fed33ff 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -332,6 +332,15 @@ Available static markers .. versionadded:: 3.7 +.. c:function:: audit(str event, void *tuple) + + Fires when :func:`sys.audit` or :c:func:`PySys_Audit` is called. + ``arg0`` is the event name as C string, ``arg1`` is a :c:type:`PyObject` + pointer to a tuple object. + + .. versionadded:: 3.8 + + SystemTap Tapsets ----------------- diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 4956aa0dd957bc..71f9fc920fdfc9 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -579,7 +579,7 @@ information. When you call one of the logging methods on an instance of information in the delegated call. Here's a snippet from the code of :class:`LoggerAdapter`:: - def debug(self, msg, *args, **kwargs): + def debug(self, msg, /, *args, **kwargs): """ Delegate a debug call to the underlying logger, after adding contextual information from this adapter instance. @@ -1079,7 +1079,7 @@ call ``str()`` on that object to get the actual format string. Consider the following two classes:: class BraceMessage: - def __init__(self, fmt, *args, **kwargs): + def __init__(self, fmt, /, *args, **kwargs): self.fmt = fmt self.args = args self.kwargs = kwargs @@ -1088,7 +1088,7 @@ following two classes:: return self.fmt.format(*self.args, **self.kwargs) class DollarMessage: - def __init__(self, fmt, **kwargs): + def __init__(self, fmt, /, **kwargs): self.fmt = fmt self.kwargs = kwargs @@ -1143,7 +1143,7 @@ to the above, as in the following example:: import logging - class Message(object): + class Message: def __init__(self, fmt, args): self.fmt = fmt self.args = args @@ -1155,7 +1155,7 @@ to the above, as in the following example:: def __init__(self, logger, extra=None): super(StyleAdapter, self).__init__(logger, extra or {}) - def log(self, level, msg, *args, **kwargs): + def log(self, level, msg, /, *args, **kwargs): if self.isEnabledFor(level): msg, kwargs = self.process(msg, kwargs) self.logger._log(level, Message(msg, args), (), **kwargs) @@ -1301,7 +1301,7 @@ You can also subclass :class:`QueueListener` to get messages from other kinds of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: class ZeroMQSocketListener(QueueListener): - def __init__(self, uri, *handlers, **kwargs): + def __init__(self, uri, /, *handlers, **kwargs): self.ctx = kwargs.get('ctx') or zmq.Context() socket = zmq.Socket(self.ctx, zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, '') # subscribe to everything @@ -1706,8 +1706,8 @@ which uses JSON to serialise the event in a machine-parseable manner:: import json import logging - class StructuredMessage(object): - def __init__(self, message, **kwargs): + class StructuredMessage: + def __init__(self, message, /, **kwargs): self.message = message self.kwargs = kwargs @@ -1750,8 +1750,8 @@ as in the following complete example:: return o.encode('unicode_escape').decode('ascii') return super(Encoder, self).default(o) - class StructuredMessage(object): - def __init__(self, message, **kwargs): + class StructuredMessage: + def __init__(self, message, /, **kwargs): self.message = message self.kwargs = kwargs @@ -1982,8 +1982,8 @@ object as a message format string, and that the logging package will call :func:`str` on that object to get the actual format string. Consider the following two classes:: - class BraceMessage(object): - def __init__(self, fmt, *args, **kwargs): + class BraceMessage: + def __init__(self, fmt, /, *args, **kwargs): self.fmt = fmt self.args = args self.kwargs = kwargs @@ -1991,8 +1991,8 @@ following two classes:: def __str__(self): return self.fmt.format(*self.args, **self.kwargs) - class DollarMessage(object): - def __init__(self, fmt, **kwargs): + class DollarMessage: + def __init__(self, fmt, /, **kwargs): self.fmt = fmt self.kwargs = kwargs @@ -2457,7 +2457,7 @@ scope of the context manager:: import logging import sys - class LoggingContext(object): + class LoggingContext: def __init__(self, logger, level=None, handler=None, close=True): self.logger = logger self.level = level diff --git a/Doc/includes/sqlite3/adapter_datetime.py b/Doc/includes/sqlite3/adapter_datetime.py index be33395100c325..d5221d80c35c8a 100644 --- a/Doc/includes/sqlite3/adapter_datetime.py +++ b/Doc/includes/sqlite3/adapter_datetime.py @@ -13,3 +13,5 @@ def adapt_datetime(ts): now = datetime.datetime.now() cur.execute("select ?", (now,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/adapter_point_1.py b/Doc/includes/sqlite3/adapter_point_1.py index 6b1af8415648a4..77daf8f16d227b 100644 --- a/Doc/includes/sqlite3/adapter_point_1.py +++ b/Doc/includes/sqlite3/adapter_point_1.py @@ -14,3 +14,5 @@ def __conform__(self, protocol): p = Point(4.0, -3.2) cur.execute("select ?", (p,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/adapter_point_2.py b/Doc/includes/sqlite3/adapter_point_2.py index d670700f0491b1..cb86331692b61d 100644 --- a/Doc/includes/sqlite3/adapter_point_2.py +++ b/Doc/includes/sqlite3/adapter_point_2.py @@ -15,3 +15,5 @@ def adapt_point(point): p = Point(4.0, -3.2) cur.execute("select ?", (p,)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/connect_db_1.py b/Doc/includes/sqlite3/connect_db_1.py deleted file mode 100644 index 1b975232865aef..00000000000000 --- a/Doc/includes/sqlite3/connect_db_1.py +++ /dev/null @@ -1,3 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") diff --git a/Doc/includes/sqlite3/connect_db_2.py b/Doc/includes/sqlite3/connect_db_2.py deleted file mode 100644 index f9728b36135ed7..00000000000000 --- a/Doc/includes/sqlite3/connect_db_2.py +++ /dev/null @@ -1,3 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") diff --git a/Doc/includes/sqlite3/countcursors.py b/Doc/includes/sqlite3/countcursors.py index ef3e70a2a9cfc5..112f47703a2ff4 100644 --- a/Doc/includes/sqlite3/countcursors.py +++ b/Doc/includes/sqlite3/countcursors.py @@ -13,3 +13,5 @@ def cursor(self, *args, **kwargs): cur1 = con.cursor() cur2 = con.cursor() print(con.numcursors) + +con.close() diff --git a/Doc/includes/sqlite3/ctx_manager.py b/Doc/includes/sqlite3/ctx_manager.py index 7af4ad1ecfbcca..6db77d45046e1f 100644 --- a/Doc/includes/sqlite3/ctx_manager.py +++ b/Doc/includes/sqlite3/ctx_manager.py @@ -14,3 +14,7 @@ con.execute("insert into person(firstname) values (?)", ("Joe",)) except sqlite3.IntegrityError: print("couldn't add Joe twice") + +# Connection object used as context manager only commits or rollbacks transactions, +# so the connection object should be closed manually +con.close() diff --git a/Doc/includes/sqlite3/execsql_fetchonerow.py b/Doc/includes/sqlite3/execsql_fetchonerow.py index 078873bfc979a9..115bcb50c7c754 100644 --- a/Doc/includes/sqlite3/execsql_fetchonerow.py +++ b/Doc/includes/sqlite3/execsql_fetchonerow.py @@ -15,3 +15,5 @@ cur.execute(SELECT) for row in cur: print('%s is %d years old.' % (row[0], row[1])) + +con.close() diff --git a/Doc/includes/sqlite3/execsql_printall_1.py b/Doc/includes/sqlite3/execsql_printall_1.py index a4ce5c528149c5..19306e6e3ca7d1 100644 --- a/Doc/includes/sqlite3/execsql_printall_1.py +++ b/Doc/includes/sqlite3/execsql_printall_1.py @@ -11,3 +11,5 @@ # Retrieve all rows as a sequence and print that sequence: print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/execute_1.py b/Doc/includes/sqlite3/execute_1.py index f864a8984e4e90..3466b1265a5bf2 100644 --- a/Doc/includes/sqlite3/execute_1.py +++ b/Doc/includes/sqlite3/execute_1.py @@ -14,3 +14,5 @@ cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age}) print(cur.fetchone()) + +con.close() diff --git a/Doc/includes/sqlite3/execute_3.py b/Doc/includes/sqlite3/execute_3.py deleted file mode 100644 index 0353683fc70476..00000000000000 --- a/Doc/includes/sqlite3/execute_3.py +++ /dev/null @@ -1,12 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() - -who = "Yeltsin" -age = 72 - -cur.execute("select name_last, age from people where name_last=:who and age=:age", - locals()) -print(cur.fetchone()) diff --git a/Doc/includes/sqlite3/executemany_1.py b/Doc/includes/sqlite3/executemany_1.py index efae10637c7e8f..edf6f8b7ebe61a 100644 --- a/Doc/includes/sqlite3/executemany_1.py +++ b/Doc/includes/sqlite3/executemany_1.py @@ -22,3 +22,5 @@ def __next__(self): cur.execute("select c from characters") print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/executemany_2.py b/Doc/includes/sqlite3/executemany_2.py index 527358ebc28e1f..02a594c861e15b 100644 --- a/Doc/includes/sqlite3/executemany_2.py +++ b/Doc/includes/sqlite3/executemany_2.py @@ -13,3 +13,5 @@ def char_generator(): cur.execute("select c from characters") print(cur.fetchall()) + +con.close() diff --git a/Doc/includes/sqlite3/executescript.py b/Doc/includes/sqlite3/executescript.py index 7e5358178d4c0a..aea8943fbee598 100644 --- a/Doc/includes/sqlite3/executescript.py +++ b/Doc/includes/sqlite3/executescript.py @@ -22,3 +22,4 @@ 1987 ); """) +con.close() diff --git a/Doc/includes/sqlite3/insert_more_people.py b/Doc/includes/sqlite3/insert_more_people.py index edbc79e7e5b6cc..10cf937243f6da 100644 --- a/Doc/includes/sqlite3/insert_more_people.py +++ b/Doc/includes/sqlite3/insert_more_people.py @@ -14,3 +14,5 @@ # The changes will not be saved unless the transaction is committed explicitly: con.commit() + +con.close() diff --git a/Doc/includes/sqlite3/load_extension.py b/Doc/includes/sqlite3/load_extension.py index b997c70668ace4..624cfe262f38b3 100644 --- a/Doc/includes/sqlite3/load_extension.py +++ b/Doc/includes/sqlite3/load_extension.py @@ -24,3 +24,5 @@ """) for row in con.execute("select rowid, name, ingredients from recipe where name match 'pie'"): print(row) + +con.close() diff --git a/Doc/includes/sqlite3/md5func.py b/Doc/includes/sqlite3/md5func.py index 0056b2d6ce84ef..16dc348bf001e2 100644 --- a/Doc/includes/sqlite3/md5func.py +++ b/Doc/includes/sqlite3/md5func.py @@ -9,3 +9,5 @@ def md5sum(t): cur = con.cursor() cur.execute("select md5(?)", (b"foo",)) print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/mysumaggr.py b/Doc/includes/sqlite3/mysumaggr.py index d2dfd2c0b98d3b..11f96395b6c485 100644 --- a/Doc/includes/sqlite3/mysumaggr.py +++ b/Doc/includes/sqlite3/mysumaggr.py @@ -18,3 +18,5 @@ def finalize(self): cur.execute("insert into test(i) values (2)") cur.execute("select mysum(i) from test") print(cur.fetchone()[0]) + +con.close() diff --git a/Doc/includes/sqlite3/parse_colnames.py b/Doc/includes/sqlite3/parse_colnames.py index cc68c76459ecea..5f01dbfe1cb524 100644 --- a/Doc/includes/sqlite3/parse_colnames.py +++ b/Doc/includes/sqlite3/parse_colnames.py @@ -6,3 +6,5 @@ cur.execute('select ? as "x [timestamp]"', (datetime.datetime.now(),)) dt = cur.fetchone()[0] print(dt, type(dt)) + +con.close() diff --git a/Doc/includes/sqlite3/pysqlite_datetime.py b/Doc/includes/sqlite3/pysqlite_datetime.py index 68d49358a578b3..5d843f906b3062 100644 --- a/Doc/includes/sqlite3/pysqlite_datetime.py +++ b/Doc/includes/sqlite3/pysqlite_datetime.py @@ -18,3 +18,5 @@ row = cur.fetchone() print("current_date", row[0], type(row[0])) print("current_timestamp", row[1], type(row[1])) + +con.close() diff --git a/Doc/includes/sqlite3/row_factory.py b/Doc/includes/sqlite3/row_factory.py index e436ffc6c80225..9de6e7b1b9052a 100644 --- a/Doc/includes/sqlite3/row_factory.py +++ b/Doc/includes/sqlite3/row_factory.py @@ -11,3 +11,5 @@ def dict_factory(cursor, row): cur = con.cursor() cur.execute("select 1 as a") print(cur.fetchone()["a"]) + +con.close() diff --git a/Doc/includes/sqlite3/rowclass.py b/Doc/includes/sqlite3/rowclass.py index 92b5ad60cb5791..fc60287069a854 100644 --- a/Doc/includes/sqlite3/rowclass.py +++ b/Doc/includes/sqlite3/rowclass.py @@ -10,3 +10,5 @@ assert row["name"] == row["nAmE"] assert row[1] == row["age"] assert row[1] == row["AgE"] + +con.close() diff --git a/Doc/includes/sqlite3/shortcut_methods.py b/Doc/includes/sqlite3/shortcut_methods.py index 71600d4f60c55e..98a39411495cba 100644 --- a/Doc/includes/sqlite3/shortcut_methods.py +++ b/Doc/includes/sqlite3/shortcut_methods.py @@ -18,3 +18,7 @@ print(row) print("I just deleted", con.execute("delete from person").rowcount, "rows") + +# close is not a shortcut method and it's not called automatically, +# so the connection object should be closed manually +con.close() diff --git a/Doc/includes/sqlite3/simple_tableprinter.py b/Doc/includes/sqlite3/simple_tableprinter.py index 231d8726cd436e..148a1707f948bc 100644 --- a/Doc/includes/sqlite3/simple_tableprinter.py +++ b/Doc/includes/sqlite3/simple_tableprinter.py @@ -24,3 +24,5 @@ print(fieldValue.ljust(FIELD_MAX_WIDTH), end=' ') print() # Finish the row with a newline. + +con.close() diff --git a/Doc/includes/sqlite3/text_factory.py b/Doc/includes/sqlite3/text_factory.py index 5f96cdb58da1ab..a857a155cdd4ff 100644 --- a/Doc/includes/sqlite3/text_factory.py +++ b/Doc/includes/sqlite3/text_factory.py @@ -25,3 +25,5 @@ cur.execute("select ?", ("bar",)) row = cur.fetchone() assert row[0] == "barfoo" + +con.close() diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index 9f47899a198e01..9ada03cfc4a4cb 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -6,7 +6,7 @@ typedef struct _typeobject { /* Methods to implement standard operations */ destructor tp_dealloc; - printfunc tp_print; + Py_ssize_t tp_vectorcall_offset; getattrfunc tp_getattr; setattrfunc tp_setattr; PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index d7814f218b502f..bd653ab32bb9c4 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -43,18 +43,32 @@ This module defines the following constants and functions: .. function:: start_new_thread(function, args[, kwargs]) - Start a new thread and return its identifier. The thread executes the function - *function* with the argument list *args* (which must be a tuple). The optional - *kwargs* argument specifies a dictionary of keyword arguments. When the function - returns, the thread silently exits. When the function terminates with an - unhandled exception, a stack trace is printed and then the thread exits (but - other threads continue to run). + Start a new thread and return its identifier. The thread executes the + function *function* with the argument list *args* (which must be a tuple). + The optional *kwargs* argument specifies a dictionary of keyword arguments. + + When the function returns, the thread silently exits. + + When the function terminates with an unhandled exception, + :func:`sys.unraisablehook` is called to handle the exception. The *object* + attribute of the hook argument is *function*. By default, a stack trace is + printed and then the thread exits (but other threads continue to run). + + When the function raises a :exc:`SystemExit` exception, it is silently + ignored. + + .. versionchanged:: 3.8 + :func:`sys.unraisablehook` is now used to handle unhandled exceptions. .. function:: interrupt_main() - Raise a :exc:`KeyboardInterrupt` exception in the main thread. A subthread can - use this function to interrupt the main thread. + Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main + thread. A thread can use this function to interrupt the main thread. + + If :data:`signal.SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. .. function:: exit() @@ -92,7 +106,7 @@ This module defines the following constants and functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. .. versionadded:: 3.8 diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index cef197f3055581..b77a38ccd48577 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -797,6 +797,15 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(['--version']) PROG 2.0 +* ``'extend'`` - This stores a list, and extends each argument value to the + list. + Example usage:: + + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument("--foo", action="extend", nargs="+", type=str) + >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"]) + Namespace(foo=['f1', 'f2', 'f3', 'f4']) + You may also specify an arbitrary action by passing an Action subclass or other object that implements the same interface. The recommended way to do this is to extend :class:`Action`, overriding the ``__call__`` method diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 4ac7bb5391a7a4..1f95dd61b9fcd7 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -83,6 +83,7 @@ The module defines the following type: to add initial items to the array. Otherwise, the iterable initializer is passed to the :meth:`extend` method. + .. audit-event:: array.__new__ "typecode initializer" .. data:: typecodes diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 1884bea80e8047..1e718382589c48 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -126,7 +126,7 @@ The abstract grammar is currently defined as follows: Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. @@ -145,11 +145,12 @@ and classes for traversing abstract syntax trees: modified to correspond to :pep:`484` "signature type comments", e.g. ``(str, int) -> List[str]``. - Also, setting ``feature_version`` to the minor version of an - earlier Python 3 version will attempt to parse using that version's - grammar. For example, setting ``feature_version=4`` will allow - the use of ``async`` and ``await`` as variable names. The lowest - supported value is 4; the highest is ``sys.version_info[1]``. + Also, setting ``feature_version`` to a tuple ``(major, minor)`` + will attempt to parse using that Python version's grammar. + Currently ``major`` must equal to ``3``. For example, setting + ``feature_version=(3, 4)`` will allow the use of ``async`` and + ``await`` as variable names. The lowest supported version is + ``(3, 4)``; the highest is ``sys.version_info[0:2]``. .. warning:: It is possible to crash the Python interpreter with a diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 06f673be7902c2..8673f84e96382e 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -504,8 +504,6 @@ Opening network connections transport. If specified, *local_addr* and *remote_addr* should be omitted (must be :const:`None`). - On Windows, with :class:`ProactorEventLoop`, this method is not supported. - See :ref:`UDP echo client protocol ` and :ref:`UDP echo server protocol ` examples. @@ -513,6 +511,9 @@ Opening network connections The *family*, *proto*, *flags*, *reuse_address*, *reuse_port, *allow_broadcast*, and *sock* parameters were added. + .. versionchanged:: 3.8 + Added support for Windows. + .. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ path=None, \*, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) @@ -1217,32 +1218,52 @@ async/await code consider using the high-level Other parameters: - * *stdin*: either a file-like object representing a pipe to be - connected to the subprocess's standard input stream using - :meth:`~loop.connect_write_pipe`, or the - :const:`subprocess.PIPE` constant (default). By default a new - pipe will be created and connected. - - * *stdout*: either a file-like object representing the pipe to be - connected to the subprocess's standard output stream using - :meth:`~loop.connect_read_pipe`, or the - :const:`subprocess.PIPE` constant (default). By default a new pipe - will be created and connected. - - * *stderr*: either a file-like object representing the pipe to be - connected to the subprocess's standard error stream using - :meth:`~loop.connect_read_pipe`, or one of - :const:`subprocess.PIPE` (default) or :const:`subprocess.STDOUT` - constants. - - By default a new pipe will be created and connected. When - :const:`subprocess.STDOUT` is specified, the subprocess' standard - error stream will be connected to the same pipe as the standard - output stream. + * *stdin* can be any of these: + + * a file-like object representing a pipe to be connected to the + subprocess's standard input stream using + :meth:`~loop.connect_write_pipe` + * the :const:`subprocess.PIPE` constant (default) which will create a new + pipe and connect it, + * the value ``None`` which will make the subprocess inherit the file + descriptor from this process + * the :const:`subprocess.DEVNULL` constant which indicates that the + special :data:`os.devnull` file will be used + + * *stdout* can be any of these: + + * a file-like object representing a pipe to be connected to the + subprocess's standard output stream using + :meth:`~loop.connect_write_pipe` + * the :const:`subprocess.PIPE` constant (default) which will create a new + pipe and connect it, + * the value ``None`` which will make the subprocess inherit the file + descriptor from this process + * the :const:`subprocess.DEVNULL` constant which indicates that the + special :data:`os.devnull` file will be used + + * *stderr* can be any of these: + + * a file-like object representing a pipe to be connected to the + subprocess's standard error stream using + :meth:`~loop.connect_write_pipe` + * the :const:`subprocess.PIPE` constant (default) which will create a new + pipe and connect it, + * the value ``None`` which will make the subprocess inherit the file + descriptor from this process + * the :const:`subprocess.DEVNULL` constant which indicates that the + special :data:`os.devnull` file will be used + * the :const:`subprocess.STDOUT` constant which will connect the standard + error stream to the process' standard output stream * All other keyword arguments are passed to :class:`subprocess.Popen` - without interpretation, except for *bufsize*, *universal_newlines* - and *shell*, which should not be specified at all. + without interpretation, except for *bufsize*, *universal_newlines*, + *shell*, *text*, *encoding* and *errors*, which should not be specified + at all. + + The ``asyncio`` subprocess API does not support decoding the streams + as text. :func:`bytes.decode` can be used to convert the bytes returned + from the stream to text. See the constructor of the :class:`subprocess.Popen` class for documentation on other arguments. diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index 81d840e23277ea..7e4a70f91c6ed5 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -53,9 +53,6 @@ All event loops on Windows do not support the following methods: :class:`ProactorEventLoop` has the following limitations: -* The :meth:`loop.create_datagram_endpoint` method - is not supported. - * The :meth:`loop.add_reader` and :meth:`loop.add_writer` methods are not supported. diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index e735b81f234d26..28ca5d5f339692 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -67,6 +67,10 @@ and work with streams: The *ssl_handshake_timeout* parameter. + .. deprecated-removed:: 3.8 3.10 + + `open_connection()` is deprecated in favor of `connect()`. + .. coroutinefunction:: start_server(client_connected_cb, host=None, \ port=None, \*, loop=None, limit=None, \ family=socket.AF_UNSPEC, \ @@ -100,6 +104,10 @@ and work with streams: The *ssl_handshake_timeout* and *start_serving* parameters. + .. deprecated-removed:: 3.8 3.10 + + `start_server()` is deprecated if favor of `StreamServer()` + .. rubric:: Unix Sockets @@ -124,6 +132,10 @@ and work with streams: The *path* parameter can now be a :term:`path-like object` + .. deprecated-removed:: 3.8 3.10 + + `open_unix_connection()` is deprecated if favor of `connect_unix()`. + .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ \*, loop=None, limit=None, sock=None, \ @@ -146,6 +158,10 @@ and work with streams: The *path* parameter can now be a :term:`path-like object`. + .. deprecated-removed:: 3.8 3.10 + + `start_unix_server()` is deprecated in favor of `UnixStreamServer()`. + --------- diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index e3f18ccb4341fe..79f6b02d85e2e5 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -66,6 +66,13 @@ Lock This method waits until the lock is *unlocked*, sets it to *locked* and returns ``True``. + When more than one coroutine is blocked in :meth:`acquire` + waiting for the lock to be unlocked, only one coroutine + eventually proceeds. + + Acquiring a lock is *fair*: the coroutine that proceeds will be + the first coroutine that started waiting on the lock. + .. method:: release() Release the lock. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index e7cf39b2bccd5f..1fcdcb985d8842 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -40,7 +40,7 @@ be executed:: >>> main() -To actually run a coroutine asyncio provides three main mechanisms: +To actually run a coroutine, asyncio provides three main mechanisms: * The :func:`asyncio.run` function to run the top-level entry point "main()" function (see the above example.) @@ -279,8 +279,8 @@ Sleeping ``sleep()`` always suspends the current task, allowing other tasks to run. - The *loop* argument is deprecated and scheduled for removal - in Python 3.10. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. .. _asyncio_example_sleep: @@ -437,8 +437,8 @@ Timeouts If the wait is cancelled, the future *aw* is also cancelled. - The *loop* argument is deprecated and scheduled for removal - in Python 3.10. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. .. _asyncio_example_waitfor: @@ -478,10 +478,12 @@ Waiting Primitives set concurrently and block until the condition specified by *return_when*. - If any awaitable in *aws* is a coroutine, it is automatically - scheduled as a Task. Passing coroutines objects to - ``wait()`` directly is deprecated as it leads to - :ref:`confusing behavior `. + .. deprecated:: 3.8 + + If any awaitable in *aws* is a coroutine, it is automatically + scheduled as a Task. Passing coroutines objects to + ``wait()`` directly is deprecated as it leads to + :ref:`confusing behavior `. Returns two sets of Tasks/Futures: ``(done, pending)``. @@ -489,8 +491,8 @@ Waiting Primitives done, pending = await asyncio.wait(aws) - The *loop* argument is deprecated and scheduled for removal - in Python 3.10. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. *timeout* (a float or int), if specified, can be used to control the maximum number of seconds to wait before returning. @@ -550,6 +552,8 @@ Waiting Primitives if task in done: # Everything will work as expected now. + .. deprecated:: 3.8 + Passing coroutine objects to ``wait()`` directly is deprecated. @@ -838,6 +842,12 @@ Task Object The *file* argument is an I/O stream to which the output is written; by default output is written to :data:`sys.stderr`. + .. method:: get_coro() + + Return the coroutine object wrapped by the :class:`Task`. + + .. versionadded:: 3.8 + .. method:: get_name() Return the name of the Task. @@ -868,8 +878,10 @@ Task Object If *loop* is ``None``, the :func:`get_event_loop` function is used to get the current loop. - This method is **deprecated** and will be removed in - Python 3.9. Use the :func:`asyncio.all_tasks` function instead. + .. deprecated-removed:: 3.7 3.9 + + Do not call this as a task method. Use the :func:`asyncio.all_tasks` + function instead. .. classmethod:: current_task(loop=None) @@ -878,9 +890,10 @@ Task Object If *loop* is ``None``, the :func:`get_event_loop` function is used to get the current loop. - This method is **deprecated** and will be removed in - Python 3.9. Use the :func:`asyncio.current_task` function - instead. + .. deprecated-removed:: 3.7 3.9 + + Do not call this as a task method. Use the + :func:`asyncio.current_task` function instead. .. _asyncio_generator_based_coro: diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 116ffcf88e3b69..7e4066cd436ad5 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -343,7 +343,7 @@ The :mod:`bdb` module also defines two classes: For backwards compatibility. Calls the :meth:`run` method. - .. method:: runcall(func, *args, **kwds) + .. method:: runcall(func, /, *args, **kwds) Debug a single function call, and return its result. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 89ecddc7780fa3..98d8679fa3dcda 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -145,8 +145,8 @@ The :mod:`binascii` module defines the following functions: platforms, use ``crc32(data) & 0xffffffff``. -.. function:: b2a_hex(data) - hexlify(data) +.. function:: b2a_hex(data[, sep[, bytes_per_sep=1]]) + hexlify(data[, sep[, bytes_per_sep=1]]) Return the hexadecimal representation of the binary *data*. Every byte of *data* is converted into the corresponding 2-digit hex representation. The @@ -155,6 +155,24 @@ The :mod:`binascii` module defines the following functions: Similar functionality (but returning a text string) is also conveniently accessible using the :meth:`bytes.hex` method. + If *sep* is specified, it must be a single character str or bytes object. + It will be inserted in the output after every *bytes_per_sep* input bytes. + Separator placement is counted from the right end of the output by default, + if you wish to count from the left, supply a negative *bytes_per_sep* value. + + >>> import binascii + >>> binascii.b2a_hex(b'\xb9\x01\xef') + b'b901ef' + >>> binascii.hexlify(b'\xb9\x01\xef', '-') + b'b9-01-ef' + >>> binascii.b2a_hex(b'\xb9\x01\xef', b'_', 2) + b'b9_01ef' + >>> binascii.b2a_hex(b'\xb9\x01\xef', b' ', -2) + b'b901 ef' + + .. versionchanged:: 3.8 + The *sep* and *bytes_per_sep* parameters were added. + .. function:: a2b_hex(hexstr) unhexlify(hexstr) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 2e9314e0fab793..5048621bf0ad14 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1198,7 +1198,8 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | mac_iceland | maciceland | Icelandic | +-----------------+--------------------------------+--------------------------------+ -| mac_latin2 | maclatin2, maccentraleurope | Central and Eastern Europe | +| mac_latin2 | maclatin2, maccentraleurope, | Central and Eastern Europe | +| | mac_centeuro | | +-----------------+--------------------------------+--------------------------------+ | mac_roman | macroman, macintosh | Western Europe | +-----------------+--------------------------------+--------------------------------+ diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index e0469c20810006..90a3f4bea9a45b 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -33,10 +33,10 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`, :class:`UserString` wrapper around string objects for easier string subclassing ===================== ==================================================================== -.. versionchanged:: 3.3 +.. deprecated-removed:: 3.3 3.9 Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module. For backwards compatibility, they continue to be visible in this module through - Python 3.7. Subsequently, they will be removed entirely. + Python 3.8. :class:`ChainMap` objects @@ -1017,15 +1017,6 @@ fields: .. versionchanged:: 3.5 Property docstrings became writeable. -Default values can be implemented by using :meth:`~somenamedtuple._replace` to -customize a prototype instance: - - >>> Account = namedtuple('Account', 'owner balance transaction_count') - >>> default_account = Account('', 0.0, 0) - >>> johns_account = default_account._replace(owner='John') - >>> janes_account = default_account._replace(owner='Jane') - - .. seealso:: * See :class:`typing.NamedTuple` for a way to add type hints for named @@ -1149,7 +1140,7 @@ variants of :func:`functools.lru_cache`:: class LRU(OrderedDict): 'Limit size, evicting the least recently looked-up key when full' - def __init__(self, maxsize=128, *args, **kwds): + def __init__(self, maxsize=128, /, *args, **kwds): self.maxsize = maxsize super().__init__(*args, **kwds) diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index 1360c7cc9f1db4..b672a05b39145d 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -21,7 +21,7 @@ spaces, the coordinates are all between 0 and 1. .. seealso:: More information about color spaces can be found at - http://poynton.ca/ColorFAQ.html and + https://poynton.ca/ColorFAQ.html and https://www.cambridgeincolour.com/tutorials/color-spaces.htm. The :mod:`colorsys` module defines the following functions: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index ffc29d782ec026..d71f2d80c9e2d0 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -28,7 +28,7 @@ Executor Objects An abstract class that provides methods to execute calls asynchronously. It should not be used directly, but through its concrete subclasses. - .. method:: submit(fn, *args, **kwargs) + .. method:: submit(fn, /, *args, **kwargs) Schedules the callable, *fn*, to be executed as ``fn(*args **kwargs)`` and returns a :class:`Future` object representing the execution of the @@ -159,6 +159,15 @@ And:: .. versionchanged:: 3.7 Added the *initializer* and *initargs* arguments. + .. versionchanged:: 3.8 + Default value of *max_workers* is changed to ``min(32, os.cpu_count() + 4)``. + This default value preserves at least 5 workers for I/O bound tasks. + It utilizes at most 32 CPU cores for CPU bound tasks which release the GIL. + And it avoids using very large resources implicitly on many-core machines. + + ThreadPoolExecutor now reuses idle worker threads before starting + *max_workers* worker threads too. + .. _threadpoolexecutor-example: @@ -297,9 +306,10 @@ The :class:`Future` class encapsulates the asynchronous execution of a callable. .. method:: cancel() - Attempt to cancel the call. If the call is currently being executed and - cannot be cancelled then the method will return ``False``, otherwise the - call will be cancelled and the method will return ``True``. + Attempt to cancel the call. If the call is currently being executed or + finished running and cannot be cancelled then the method will return + ``False``, otherwise the call will be cancelled and the method will + return ``True``. .. method:: cancelled() @@ -414,8 +424,9 @@ Module Functions Wait for the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* to complete. Returns a named 2-tuple of sets. The first set, named ``done``, contains the futures that - completed (finished or were cancelled) before the wait completed. The second - set, named ``not_done``, contains uncompleted futures. + completed (finished or cancelled futures) before the wait completed. The + second set, named ``not_done``, contains the futures that did not complete + (pending or running futures). *timeout* can be used to control the maximum number of seconds to wait before returning. *timeout* can be an int or float. If *timeout* is not specified @@ -446,7 +457,7 @@ Module Functions Returns an iterator over the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or were cancelled). Any futures given by *fs* that + they complete (finished or cancelled futures). Any futures given by *fs* that are duplicated will be returned once. Any futures that completed before :func:`as_completed` is called will be yielded first. The returned iterator raises a :exc:`concurrent.futures.TimeoutError` if :meth:`~iterator.__next__` diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 017a87a5648c39..0aa4ad76523480 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -416,7 +416,7 @@ Functions and classes provided: The passed in object is returned from the function, allowing this method to be used as a function decorator. - .. method:: callback(callback, *args, **kwds) + .. method:: callback(callback, /, *args, **kwds) Accepts an arbitrary callback function and arguments and adds it to the callback stack. @@ -473,7 +473,7 @@ Functions and classes provided: Similar to :meth:`push` but expects either an asynchronous context manager or a coroutine function. - .. method:: push_async_callback(callback, *args, **kwds) + .. method:: push_async_callback(callback, /, *args, **kwds) Similar to :meth:`callback` but expects a coroutine function. @@ -637,7 +637,7 @@ even further by means of a small helper class:: from contextlib import ExitStack class Callback(ExitStack): - def __init__(self, callback, *args, **kwds): + def __init__(self, callback, /, *args, **kwds): super(Callback, self).__init__() self.callback(callback, *args, **kwds) diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 40fca56d8029e9..43920210095951 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -49,7 +49,7 @@ The example below would like to show how to register a pickle function and how it will be used: >>> import copyreg, copy, pickle - >>> class C(object): + >>> class C: ... def __init__(self, a): ... self.a = a ... diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 1c60b4bbda13ec..97172c588ae97f 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1509,6 +1509,17 @@ object is available: :c:type:`int`, which is of course not always the truth, so you have to assign the correct :attr:`restype` attribute to use these functions. +.. audit-event:: ctypes.dlopen name + + Loading a library through any of these objects raises an + :ref:`auditing event ` ``ctypes.dlopen`` with string argument + ``name``, the name used to load the library. + +.. audit-event:: ctypes.dlsym "library name" + + Accessing a function on a loaded library raises an auditing event + ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name`` + (the symbol's name as a string or integer). .. _ctypes-foreign-functions: @@ -2032,6 +2043,12 @@ Data types This method returns a ctypes type instance using the memory specified by *address* which must be an integer. + .. audit-event:: ctypes.cdata address + + This method, and others that indirectly call this method, raises an + :func:`auditing event ` ``ctypes.cdata`` with argument + ``address``. + .. method:: from_param(obj) This method adapts *obj* to a ctypes type. It is called with the actual diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 7d1e7538a292b3..c88d3520e988ff 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -656,7 +656,7 @@ The module :mod:`curses` defines the following functions: foreground color on the default background. -.. function:: wrapper(func, ...) +.. function:: wrapper(func, /, *args, **kwargs) Initialize curses and call another callable object, *func*, which should be the rest of your curses-using application. If the application raises an exception, diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index abdc977354803e..fb41aeeb5bed03 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1365,56 +1365,64 @@ Examples of working with datetime objects: Using datetime with tzinfo: - >>> from datetime import timedelta, datetime, tzinfo - >>> class GMT1(tzinfo): + >>> from datetime import timedelta, datetime, tzinfo, timezone + >>> class KabulTz(tzinfo): + ... # Kabul used +4 until 1945, when they moved to +4:30 + ... UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) ... def utcoffset(self, dt): - ... return timedelta(hours=1) + self.dst(dt) - ... def dst(self, dt): - ... # DST starts last Sunday in March - ... d = datetime(dt.year, 4, 1) # ends last Sunday in October - ... self.dston = d - timedelta(days=d.weekday() + 1) - ... d = datetime(dt.year, 11, 1) - ... self.dstoff = d - timedelta(days=d.weekday() + 1) - ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) + ... if dt.year < 1945: + ... return timedelta(hours=4) + ... elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): + ... # If dt falls in the imaginary range, use fold to decide how + ... # to resolve. See PEP495 + ... return timedelta(hours=4, minutes=(30 if dt.fold else 0)) ... else: - ... return timedelta(0) - ... def tzname(self,dt): - ... return "GMT +1" + ... return timedelta(hours=4, minutes=30) + ... + ... def fromutc(self, dt): + ... # A custom implementation is required for fromutc as + ... # the input to this function is a datetime with utc values + ... # but with a tzinfo set to self + ... # See datetime.astimezone or fromtimestamp + ... + ... # Follow same validations as in datetime.tzinfo + ... if not isinstance(dt, datetime): + ... raise TypeError("fromutc() requires a datetime argument") + ... if dt.tzinfo is not self: + ... raise ValueError("dt.tzinfo is not self") + ... + ... if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: + ... return dt + timedelta(hours=4, minutes=30) + ... else: + ... return dt + timedelta(hours=4) ... - >>> class GMT2(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=2) + self.dst(dt) ... def dst(self, dt): - ... d = datetime(dt.year, 4, 1) - ... self.dston = d - timedelta(days=d.weekday() + 1) - ... d = datetime(dt.year, 11, 1) - ... self.dstoff = d - timedelta(days=d.weekday() + 1) - ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) + ... return timedelta(0) + ... + ... def tzname(self, dt): + ... if dt >= self.UTC_MOVE_DATE: + ... return "+04:30" ... else: - ... return timedelta(0) - ... def tzname(self,dt): - ... return "GMT +2" + ... return "+04" ... - >>> gmt1 = GMT1() - >>> # Daylight Saving Time - >>> dt1 = datetime(2006, 11, 21, 16, 30, tzinfo=gmt1) - >>> dt1.dst() - datetime.timedelta(0) - >>> dt1.utcoffset() - datetime.timedelta(seconds=3600) - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=gmt1) - >>> dt2.dst() - datetime.timedelta(seconds=3600) - >>> dt2.utcoffset() - datetime.timedelta(seconds=7200) + ... def __repr__(self): + ... return f"{self.__class__.__name__}()" + ... + >>> tz1 = KabulTz() + >>> # Datetime before the change + >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) + >>> print(dt1.utcoffset()) + 4:00:00 + >>> # Datetime after the change + >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) + >>> print(dt2.utcoffset()) + 4:30:00 >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(GMT2()) - >>> dt3 # doctest: +ELLIPSIS - datetime.datetime(2006, 6, 14, 14, 0, tzinfo=) - >>> dt2 # doctest: +ELLIPSIS - datetime.datetime(2006, 6, 14, 13, 0, tzinfo=) + >>> dt3 = dt2.astimezone(timezone.utc) + >>> dt3 + datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) + >>> dt2 + datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz()) >>> dt2.utctimetuple() == dt3.utctimetuple() True @@ -1656,26 +1664,27 @@ Instance methods: Example: >>> from datetime import time, tzinfo, timedelta - >>> class GMT1(tzinfo): + >>> class TZ1(tzinfo): ... def utcoffset(self, dt): ... return timedelta(hours=1) ... def dst(self, dt): ... return timedelta(0) ... def tzname(self,dt): - ... return "Europe/Prague" + ... return "+01:00" + ... def __repr__(self): + ... return f"{self.__class__.__name__}()" ... - >>> t = time(12, 10, 30, tzinfo=GMT1()) - >>> t # doctest: +ELLIPSIS - datetime.time(12, 10, 30, tzinfo=) - >>> gmt = GMT1() + >>> t = time(12, 10, 30, tzinfo=TZ1()) + >>> t + datetime.time(12, 10, 30, tzinfo=TZ1()) >>> t.isoformat() '12:10:30+01:00' >>> t.dst() datetime.timedelta(0) >>> t.tzname() - 'Europe/Prague' + '+01:00' >>> t.strftime("%H:%M:%S %Z") - '12:10:30 Europe/Prague' + '12:10:30 +01:00' >>> 'The {} is {:%H:%M}.'.format("time", t) 'The time is 12:10.' @@ -2048,6 +2057,9 @@ For :class:`date` objects, the format codes for hours, minutes, seconds, and microseconds should not be used, as :class:`date` objects have no such values. If they're used anyway, ``0`` is substituted for them. +For the :meth:`datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``: +any components not specified in the format string will be pulled from the default value. [#]_ + The full set of format codes supported varies across platforms, because Python calls the platform C library's :func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your @@ -2282,3 +2294,4 @@ Notes: .. rubric:: Footnotes .. [#] If, that is, we ignore the effects of Relativity +.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 060d4bb6997a4d..5b79be626626de 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -708,7 +708,7 @@ iterations of the loop. Cleans up the value stack and the block stack. If *preserve_tos* is not ``0`` TOS first is popped from the stack and pushed on the stack after - perfoming other stack operations: + performing other stack operations: * If TOS is ``NULL`` or an integer (pushed by :opcode:`BEGIN_FINALLY` or :opcode:`CALL_FINALLY`) it is popped from the stack. @@ -1113,9 +1113,13 @@ All of the following opcodes use their arguments. .. opcode:: RAISE_VARARGS (argc) - Raises an exception. *argc* indicates the number of arguments to the raise - statement, ranging from 0 to 3. The handler will find the traceback as TOS2, - the parameter as TOS1, and the exception as TOS. + Raises an exception using one of the 3 forms of the ``raise`` statement, + depending on the value of *argc*: + + * 0: ``raise`` (re-raise previous exception) + * 1: ``raise TOS`` (raise exception instance or type at ``TOS``) + * 2: ``raise TOS1 from TOS`` (raise exception instance or type at ``TOS1`` + with ``__cause__`` set to ``TOS``) .. opcode:: CALL_FUNCTION (argc) @@ -1215,10 +1219,10 @@ All of the following opcodes use their arguments. .. opcode:: EXTENDED_ARG (ext) - Prefixes any opcode which has an argument too big to fit into the default two - bytes. *ext* holds two additional bytes which, taken together with the - subsequent opcode's argument, comprise a four-byte argument, *ext* being the - two most-significant bytes. + Prefixes any opcode which has an argument too big to fit into the default one + byte. *ext* holds an additional byte which act as higher bits in the argument. + For each opcode, at most three prefixal ``EXTENDED_ARG`` are allowed, forming + an argument from two-byte to four-byte. .. opcode:: FORMAT_VALUE (flags) diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index f02b35b910a0cf..09ea64a5a01aae 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -6,6 +6,7 @@ .. module:: email.message :synopsis: The base class representing email messages in a fashion backward compatible with Python 3.2 + :noindex: The :class:`Message` class is very similar to the diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 70bf61323c39ed..e4752a5edf841f 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -12,6 +12,11 @@ This module is part of the legacy (``Compat32``) email API. In the new API the functionality is provided by the *cte* parameter of the :meth:`~email.message.EmailMessage.set_content` method. +This module is deprecated in Python 3. The functions provided here +should not be called explicitly since the :class:`~email.mime.text.MIMEText` +class sets the content type and CTE header using the *_subtype* and *_charset* +values passed during the instaniation of that class. + The remaining text in this section is the original documentation of the module. When creating :class:`~email.message.Message` objects from scratch, you often diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index c09ae8cbc60410..2d9bae6a7ee57b 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -36,7 +36,7 @@ something that contains only ASCII characters, using the standard email RFC Content Transfer Encoding techniques for encoding email messages for transport over channels that are not "8 bit clean". -To accomodate reproducible processing of SMIME-signed messages +To accommodate reproducible processing of SMIME-signed messages :class:`Generator` disables header folding for message parts of type ``multipart/signed`` and all subparts. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index ce283c6b596cf2..9376da2b8d39ce 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -107,7 +107,7 @@ headers. method if it wishes to set additional attributes beyond those provided by ``BaseHeader`` itself. Such an ``init`` method should look like this:: - def init(self, *args, **kw): + def init(self, /, *args, **kw): self._myattr = kw.pop('myattr') super().init(*args, **kw) @@ -321,19 +321,26 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. The default mappings are: - :subject: UniqueUnstructuredHeader - :date: UniqueDateHeader - :resent-date: DateHeader - :orig-date: UniqueDateHeader - :sender: UniqueSingleAddressHeader - :resent-sender: SingleAddressHeader - :to: UniqueAddressHeader - :resent-to: AddressHeader - :cc: UniqueAddressHeader - :resent-cc: AddressHeader - :from: UniqueAddressHeader - :resent-from: AddressHeader - :reply-to: UniqueAddressHeader + :subject: UniqueUnstructuredHeader + :date: UniqueDateHeader + :resent-date: DateHeader + :orig-date: UniqueDateHeader + :sender: UniqueSingleAddressHeader + :resent-sender: SingleAddressHeader + :to: UniqueAddressHeader + :resent-to: AddressHeader + :cc: UniqueAddressHeader + :resent-cc: AddressHeader + :bcc: UniqueAddressHeader + :resent-bcc: AddressHeader + :from: UniqueAddressHeader + :resent-from: AddressHeader + :reply-to: UniqueAddressHeader + :mime-version: MIMEVersionHeader + :content-type: ContentTypeHeader + :content-disposition: ContentDispositionHeader + :content-transfer-encoding: ContentTransferEncodingHeader + :message-id: MessageIDHeader ``HeaderRegistry`` has the following methods: diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 88112f6b7e545b..2db9674952d7b6 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -28,6 +28,10 @@ descriptor. Operations in this module used to raise an :exc:`IOError` where they now raise an :exc:`OSError`. +.. versionchanged:: 3.8 + The fcntl module now contains ``F_ADD_SEALS``, ``F_GET_SEALS``, and + ``F_SEAL_*`` constants for sealing of :func:`os.memfd_create` file + descriptors. The module defines the following functions: diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index af9dff34a80827..f5e5280a136399 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -23,8 +23,9 @@ The typical use is:: This iterates over the lines of all files listed in ``sys.argv[1:]``, defaulting to ``sys.stdin`` if the list is empty. If a filename is ``'-'``, it is also -replaced by ``sys.stdin``. To specify an alternative list of filenames, pass it -as the first argument to :func:`.input`. A single file name is also allowed. +replaced by ``sys.stdin`` and the optional arguments *mode* and *openhook* +are ignored. To specify an alternative list of filenames, pass it as the +first argument to :func:`.input`. A single file name is also allowed. All files are opened in text mode by default, but you can override this by specifying the *mode* parameter in the call to :func:`.input` or @@ -54,7 +55,7 @@ provided by this module. The following function is the primary interface of this module: -.. function:: input(files=None, inplace=False, backup='', bufsize=0, mode='r', openhook=None) +.. function:: input(files=None, inplace=False, backup='', *, mode='r', openhook=None) Create an instance of the :class:`FileInput` class. The instance will be used as global state for the functions of this module, and is also returned to use @@ -72,8 +73,9 @@ The following function is the primary interface of this module: .. versionchanged:: 3.2 Can be used as a context manager. - .. deprecated-removed:: 3.6 3.8 - The *bufsize* parameter. + .. versionchanged:: 3.8 + The keyword parameters *mode* and *openhook* are now keyword-only. + The following functions use the global state created by :func:`fileinput.input`; if there is no active state, :exc:`RuntimeError` is raised. @@ -135,7 +137,7 @@ The class which implements the sequence behavior provided by the module is available for subclassing as well: -.. class:: FileInput(files=None, inplace=False, backup='', bufsize=0, mode='r', openhook=None) +.. class:: FileInput(files=None, inplace=False, backup='', *, mode='r', openhook=None) Class :class:`FileInput` is the implementation; its methods :meth:`filename`, :meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`, @@ -160,18 +162,20 @@ available for subclassing as well: with FileInput(files=('spam.txt', 'eggs.txt')) as input: process(input) + .. versionchanged:: 3.2 Can be used as a context manager. .. deprecated:: 3.4 The ``'rU'`` and ``'U'`` modes. - .. deprecated-removed:: 3.6 3.8 - The *bufsize* parameter. - .. deprecated:: 3.8 Support for :meth:`__getitem__` method is deprecated. + .. versionchanged:: 3.8 + The keyword parameter *mode* and *openhook* are now keyword-only. + + **Optional in-place filtering:** if the keyword argument ``inplace=True`` is passed to :func:`fileinput.input` or to the :class:`FileInput` constructor, the diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 613e4f74ac4176..88977056723f2e 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -257,6 +257,12 @@ are always available. They are listed here in alphabetical order. can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on the :class:`~__future__._Feature` instance in the :mod:`__future__` module. + The optional argument *flags* also controls whether the compiled source is + allowed to contain top-level ``await``, ``async for`` and ``async with``. + When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the return code + object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively + executed via ``await eval(code_object)``. + The argument *optimize* specifies the optimization level of the compiler; the default value of ``-1`` selects the optimization level of the interpreter as given by :option:`-O` options. Explicit levels are ``0`` (no optimization; @@ -269,6 +275,12 @@ are always available. They are listed here in alphabetical order. If you want to parse Python code into its AST representation, see :func:`ast.parse`. + .. audit-event:: compile "source filename" + + Raises an :func:`auditing event ` ``compile`` with arguments + ``source`` and ``filename``. This event may also be raised by implicit + compilation. + .. note:: When compiling a string with multi-line code in ``'single'`` or @@ -290,6 +302,10 @@ are always available. They are listed here in alphabetical order. Previously, :exc:`TypeError` was raised when null bytes were encountered in *source*. + .. versionadded:: 3.8 + ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable + support for top-level ``await``, ``async for``, and ``async with``. + .. class:: complex([real[, imag]]) @@ -302,6 +318,11 @@ are always available. They are listed here in alphabetical order. :class:`int` and :class:`float`. If both arguments are omitted, returns ``0j``. + For a general Python object ``x``, ``complex(x)`` delegates to + ``x.__complex__()``. If ``__complex__()`` is not defined then it falls back + to :meth:`__float__`. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. + .. note:: When converting from a string, the string must not contain whitespace @@ -314,6 +335,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.6 Grouping digits with underscores as in code literals is allowed. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__complex__` and + :meth:`__float__` are not defined. + .. function:: delattr(object, name) @@ -463,6 +488,11 @@ are always available. They are listed here in alphabetical order. See :func:`ast.literal_eval` for a function that can safely evaluate strings with expressions containing only literals. + .. audit-event:: exec code_object + + Raises an :func:`auditing event ` ``exec`` with the code object as + the argument. Code compilation events may also be raised. + .. index:: builtin: exec .. function:: exec(object[, globals[, locals]]) @@ -478,7 +508,8 @@ are always available. They are listed here in alphabetical order. :func:`exec` function. The return value is ``None``. In all cases, if the optional parts are omitted, the code is executed in the - current scope. If only *globals* is provided, it must be a dictionary, which + current scope. If only *globals* is provided, it must be a dictionary + (and not a subclass of dictionary), which will be used for both the global and the local variables. If *globals* and *locals* are given, they are used for the global and local variables, respectively. If provided, *locals* can be any mapping object. Remember @@ -492,6 +523,11 @@ are always available. They are listed here in alphabetical order. builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. + .. audit-event:: exec code_object + + Raises an :func:`auditing event ` ``exec`` with the code object as + the argument. Code compilation events may also be raised. + .. note:: The built-in functions :func:`globals` and :func:`locals` return the current @@ -557,7 +593,8 @@ are always available. They are listed here in alphabetical order. float, an :exc:`OverflowError` will be raised. For a general Python object ``x``, ``float(x)`` delegates to - ``x.__float__()``. + ``x.__float__()``. If ``__float__()`` is not defined then it falls back + to :meth:`__index__`. If no argument is given, ``0.0`` is returned. @@ -582,6 +619,9 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.7 *x* is now a positional-only parameter. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__float__` is not defined. + .. index:: single: __format__ @@ -737,13 +777,24 @@ are always available. They are listed here in alphabetical order. If the :mod:`readline` module was loaded, then :func:`input` will use it to provide elaborate line editing and history features. + .. audit-event:: builtins.input prompt + + Raises an :func:`auditing event ` ``builtins.input`` with + argument ``prompt`` before reading input + + .. audit-event:: builtins.input/result result + + Raises an auditing event ``builtins.input/result`` with the result after + successfully reading input. + .. class:: int([x]) int(x, base=10) Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* defines :meth:`__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__trunc__`, + ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__index__`, + it returns ``x.__index__()``. If *x* defines :meth:`__trunc__`, it returns ``x.__trunc__()``. For floating point numbers, this truncates towards zero. @@ -775,6 +826,9 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.7 *x* is now a positional-only parameter. + .. versionchanged:: 3.8 + Falls back to :meth:`__index__` if :meth:`__int__` is not defined. + .. function:: isinstance(object, classinfo) @@ -1166,6 +1220,11 @@ are always available. They are listed here in alphabetical order. (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, and :mod:`shutil`. + .. audit-event:: open "file mode flags" + + The ``mode`` and ``flags`` arguments may have been modified or inferred from + the original call. + .. versionchanged:: 3.3 @@ -1218,9 +1277,24 @@ are always available. They are listed here in alphabetical order. operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are converted to float and a float result is delivered. For example, ``10**2`` - returns ``100``, but ``10**-2`` returns ``0.01``. If the second argument is - negative, the third argument must be omitted. If *z* is present, *x* and *y* - must be of integer types, and *y* must be non-negative. + returns ``100``, but ``10**-2`` returns ``0.01``. + + For :class:`int` operands *x* and *y*, if *z* is present, *z* must also be + of integer type and *z* must be nonzero. If *z* is present and *y* is + negative, *x* must be relatively prime to *z*. In that case, ``pow(inv_x, + -y, z)`` is returned, where *inv_x* is an inverse to *x* modulo *z*. + + Here's an example of computing an inverse for ``38`` modulo ``97``:: + + >>> pow(38, -1, 97) + 23 + >>> 23 * 38 % 97 == 1 + True + + .. versionchanged:: 3.8 + For :class:`int` operands, the three-argument form of ``pow`` now allows + the second argument to be negative, permitting computation of modular + inverses. .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) @@ -1488,11 +1562,11 @@ are always available. They are listed here in alphabetical order. about strings, see :ref:`textseq`. -.. function:: sum(iterable[, start]) +.. function:: sum(iterable, /, start=0) Sums *start* and the items of an *iterable* from left to right and returns the - total. *start* defaults to ``0``. The *iterable*'s items are normally numbers, - and the start value is not allowed to be a string. + total. The *iterable*'s items are normally numbers, and the start value is not + allowed to be a string. For some use cases, there are good alternatives to :func:`sum`. The preferred, fast way to concatenate a sequence of strings is by calling diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 16a779fa836858..d3debac8432b19 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -76,7 +76,8 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.2 -.. decorator:: lru_cache(maxsize=128, typed=False) +.. decorator:: lru_cache(user_function) + lru_cache(maxsize=128, typed=False) Decorator to wrap a function with a memoizing callable that saves up to the *maxsize* most recent calls. It can save time when an expensive or I/O bound @@ -90,6 +91,15 @@ The :mod:`functools` module defines the following functions: differ in their keyword argument order and may have two separate cache entries. + If *user_function* is specified, it must be a callable. This allows the + *lru_cache* decorator to be applied directly to a user function, leaving + the *maxsize* at its default value of 128:: + + @lru_cache + def count_vowels(sentence): + sentence = sentence.casefold() + return sum(sentence.count(vowel) for vowel in 'aeiou') + If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can grow without bound. The LRU feature performs best when *maxsize* is a power-of-two. @@ -165,6 +175,9 @@ The :mod:`functools` module defines the following functions: .. versionchanged:: 3.3 Added the *typed* option. + .. versionchanged:: 3.8 + Added the *user_function* option. + .. decorator:: total_ordering Given a class defining one or more rich comparison ordering methods, this @@ -208,7 +221,7 @@ The :mod:`functools` module defines the following functions: Returning NotImplemented from the underlying comparison function for unrecognised types is now supported. -.. function:: partial(func, *args, **keywords) +.. function:: partial(func, /, *args, **keywords) Return a new :ref:`partial object` which when called will behave like *func* called with the positional arguments *args* @@ -217,7 +230,7 @@ The :mod:`functools` module defines the following functions: supplied, they extend and override *keywords*. Roughly equivalent to:: - def partial(func, *args, **keywords): + def partial(func, /, *args, **keywords): def newfunc(*fargs, **fkeywords): newkeywords = {**keywords, **fkeywords} return func(*args, *fargs, **newkeywords) @@ -239,7 +252,7 @@ The :mod:`functools` module defines the following functions: 18 -.. class:: partialmethod(func, *args, **keywords) +.. class:: partialmethod(func, /, *args, **keywords) Return a new :class:`partialmethod` descriptor which behaves like :class:`partial` except that it is designed to be used as a method @@ -262,7 +275,7 @@ The :mod:`functools` module defines the following functions: Example:: - >>> class Cell(object): + >>> class Cell: ... def __init__(self): ... self._alive = False ... @property diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 7f4eab5843f5f2..937330bb201b08 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -39,7 +39,7 @@ class-based API instead. Bind the *domain* to the locale directory *localedir*. More concretely, :mod:`gettext` will look for binary :file:`.mo` files for the given domain using the path (on Unix): :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo`, where - *languages* is searched for in the environment variables :envvar:`LANGUAGE`, + *language* is searched for in the environment variables :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. If *localedir* is omitted or ``None``, then the current binding for *domain* is diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index f511d64b550be6..bd24695c728233 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -281,16 +281,16 @@ Configure IDLE menu. For more, see :ref:`Setting preferences ` under Help and preferences. -Zoom/Restore Height - Toggles the window between normal size and maximum height. The initial size - defaults to 40 lines by 80 chars unless changed on the General tab of the - Configure IDLE dialog. - Show/Hide Code Context (Editor Window only) Open a pane at the top of the edit window which shows the block context of the code which has scrolled above the top of the window. See :ref:`Code Context ` in the Editing and Navigation section below. +Zoom/Restore Height + Toggles the window between normal size and maximum height. The initial size + defaults to 40 lines by 80 chars unless changed on the General tab of the + Configure IDLE dialog. + Window menu (Shell and Editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -700,6 +700,9 @@ If ``sys`` is reset by user code, such as with ``importlib.reload(sys)``, IDLE's changes are lost and input from the keyboard and output to the screen will not work correctly. +When user code raises SystemExit either directly or by calling sys.exit, IDLE +returns to a Shell prompt instead of exiting. + User output in Shell ^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst new file mode 100644 index 00000000000000..2126498e728cfa --- /dev/null +++ b/Doc/library/importlib.metadata.rst @@ -0,0 +1,257 @@ +.. _using: + +========================== + Using importlib.metadata +========================== + +.. note:: + This functionality is provisional and may deviate from the usual + version semantics of the standard library. + +``importlib.metadata`` is a library that provides for access to installed +package metadata. Built in part on Python's import system, this library +intends to replace similar functionality in the `entry point +API`_ and `metadata API`_ of ``pkg_resources``. Along with +``importlib.resources`` in `Python 3.7 +and newer`_ (backported as `importlib_resources`_ for older versions of +Python), this can eliminate the need to use the older and less efficient +``pkg_resources`` package. + +By "installed package" we generally mean a third-party package installed into +Python's ``site-packages`` directory via tools such as `pip +`_. Specifically, +it means a package with either a discoverable ``dist-info`` or ``egg-info`` +directory, and metadata defined by `PEP 566`_ or its older specifications. +By default, package metadata can live on the file system or in zip archives on +``sys.path``. Through an extension mechanism, the metadata can live almost +anywhere. + + +Overview +======== + +Let's say you wanted to get the version string for a package you've installed +using ``pip``. We start by creating a virtual environment and installing +something into it: + +.. code-block:: shell-session + + $ python3 -m venv example + $ source example/bin/activate + (example) $ pip install wheel + +You can get the version string for ``wheel`` by running the following: + +.. code-block:: pycon + + (example) $ python + >>> from importlib.metadata import version # doctest: +SKIP + >>> version('wheel') # doctest: +SKIP + '0.32.3' + +You can also get the set of entry points keyed by group, such as +``console_scripts``, ``distutils.commands`` and others. Each group contains a +sequence of :ref:`EntryPoint ` objects. + +You can get the :ref:`metadata for a distribution `:: + + >>> list(metadata('wheel')) # doctest: +SKIP + ['Metadata-Version', 'Name', 'Version', 'Summary', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', 'Project-URL', 'Project-URL', 'Project-URL', 'Keywords', 'Platform', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Requires-Python', 'Provides-Extra', 'Requires-Dist', 'Requires-Dist'] + +You can also get a :ref:`distribution's version number `, list its +:ref:`constituent files `, and get a list of the distribution's +:ref:`requirements`. + + +Functional API +============== + +This package provides the following functionality via its public API. + + +.. _entry-points: + +Entry points +------------ + +The ``entry_points()`` function returns a dictionary of all entry points, +keyed by group. Entry points are represented by ``EntryPoint`` instances; +each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and +a ``.load()`` method to resolve the value. + + >>> eps = entry_points() # doctest: +SKIP + >>> list(eps) # doctest: +SKIP + ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] + >>> scripts = eps['console_scripts'] # doctest: +SKIP + >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP + >>> wheel # doctest: +SKIP + EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> main = wheel.load() # doctest: +SKIP + >>> main # doctest: +SKIP + + +The ``group`` and ``name`` are arbitrary values defined by the package author +and usually a client will wish to resolve all entry points for a particular +group. Read `the setuptools docs +`_ +for more information on entrypoints, their definition, and usage. + + +.. _metadata: + +Distribution metadata +--------------------- + +Every distribution includes some metadata, which you can extract using the +``metadata()`` function:: + + >>> wheel_metadata = metadata('wheel') # doctest: +SKIP + +The keys of the returned data structure [#f1]_ name the metadata keywords, and +their values are returned unparsed from the distribution metadata:: + + >>> wheel_metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + + +.. _version: + +Distribution versions +--------------------- + +The ``version()`` function is the quickest way to get a distribution's version +number, as a string:: + + >>> version('wheel') # doctest: +SKIP + '0.32.3' + + +.. _files: + +Distribution files +------------------ + +You can also get the full set of files contained within a distribution. The +``files()`` function takes a distribution package name and returns all of the +files installed by this distribution. Each file object returned is a +``PackagePath``, a `pathlib.Path`_ derived object with additional ``dist``, +``size``, and ``hash`` properties as indicated by the metadata. For example:: + + >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP + >>> util # doctest: +SKIP + PackagePath('wheel/util.py') + >>> util.size # doctest: +SKIP + 859 + >>> util.dist # doctest: +SKIP + + >>> util.hash # doctest: +SKIP + + +Once you have the file, you can also read its contents:: + + >>> print(util.read_text()) # doctest: +SKIP + import base64 + import sys + ... + def as_bytes(s): + if isinstance(s, text_type): + return s.encode('utf-8') + return s + + +.. _requirements: + +Distribution requirements +------------------------- + +To get the full set of requirements for a distribution, use the ``requires()`` +function. Note that this returns an iterator:: + + >>> list(requires('wheel')) # doctest: +SKIP + ["pytest (>=3.0.0) ; extra == 'test'"] + + +Distributions +============= + +While the above API is the most common and convenient usage, you can get all +of that information from the ``Distribution`` class. A ``Distribution`` is an +abstract object that represents the metadata for a Python package. You can +get the ``Distribution`` instance:: + + >>> from importlib.metadata import distribution # doctest: +SKIP + >>> dist = distribution('wheel') # doctest: +SKIP + +Thus, an alternative way to get the version number is through the +``Distribution`` instance:: + + >>> dist.version # doctest: +SKIP + '0.32.3' + +There are all kinds of additional metadata available on the ``Distribution`` +instance:: + + >>> d.metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + >>> d.metadata['License'] # doctest: +SKIP + 'MIT' + +The full set of available metadata is not described here. See `PEP 566 +`_ for additional details. + + +Extending the search algorithm +============================== + +Because package metadata is not available through ``sys.path`` searches, or +package loaders directly, the metadata for a package is found through import +system `finders`_. To find a distribution package's metadata, +``importlib.metadata`` queries the list of `meta path finders`_ on +`sys.meta_path`_. + +By default ``importlib.metadata`` installs a finder for distribution packages +found on the file system. This finder doesn't actually find any *packages*, +but it can find the packages' metadata. + +The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the +interface expected of finders by Python's import system. +``importlib.metadata`` extends this protocol by looking for an optional +``find_distributions`` callable on the finders from +``sys.meta_path``. If the finder has this method, it must return +an iterator over instances of the ``Distribution`` abstract class. This +method must have the signature:: + + def find_distributions(name=None, path=None): + """Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the name + (or all names if not supplied) along the paths in the list + of directories ``path`` (defaults to sys.path). + """ + +What this means in practice is that to support finding distribution package +metadata in locations other than the file system, you should derive from +``Distribution`` and implement the ``load_metadata()`` method. This takes a +single argument which is the name of the package whose metadata is being +found. This instance of the ``Distribution`` base abstract class is what your +finder's ``find_distributions()`` method should return. + + +.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points +.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api +.. _`Python 3.7 and newer`: https://docs.python.org/3/library/importlib.html#module-importlib.resources +.. _`importlib_resources`: https://importlib-resources.readthedocs.io/en/latest/index.html +.. _`PEP 566`: https://www.python.org/dev/peps/pep-0566/ +.. _`finders`: https://docs.python.org/3/reference/import.html#finders-and-loaders +.. _`meta path finders`: https://docs.python.org/3/glossary.html#term-meta-path-finder +.. _`sys.meta_path`: https://docs.python.org/3/library/sys.html#sys.meta_path +.. _`pathlib.Path`: https://docs.python.org/3/library/pathlib.html#pathlib.Path + + +.. rubric:: Footnotes + +.. [#f1] Technically, the returned distribution metadata object is an + `email.message.Message + `_ + instance, but this is an implementation detail, and not part of the + stable API. You should only use dictionary-like methods and syntax + to access the metadata contents. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 81824ddc1e54db..2a71201a80b2cb 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -169,6 +169,9 @@ attributes: | | | variables (referenced via | | | | a function's closure) | +-----------+-------------------+---------------------------+ +| | co_posonlyargcount| number of positional only | +| | | arguments | ++-----------+-------------------+---------------------------+ | | co_kwonlyargcount | number of keyword only | | | | arguments (not including | | | | \*\* arg) | @@ -724,13 +727,9 @@ function. | Name | Meaning | +========================+==============================================+ | *POSITIONAL_ONLY* | Value must be supplied as a positional | - | | argument. | - | | | - | | Python has no explicit syntax for defining | - | | positional-only parameters, but many built-in| - | | and extension module functions (especially | - | | those that accept only one or two parameters)| - | | accept them. | + | | argument. Positional only parameters are | + | | those which appear before a ``/`` entry (if | + | | present) in a Python function definition. | +------------------------+----------------------------------------------+ | *POSITIONAL_OR_KEYWORD*| Value may be supplied as either a keyword or | | | positional argument (this is the standard | @@ -1023,7 +1022,7 @@ Classes and functions metatype is in use, cls will be the first element of the tuple. -.. function:: getcallargs(func, *args, **kwds) +.. function:: getcallargs(func, /, *args, **kwds) Bind the *args* and *kwds* to the argument names of the Python function or method *func*, as if it was called with them. For bound methods, bind also the diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 0f1251687aeb13..2fb27c3aad78b9 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -120,6 +120,27 @@ High-level Module Interface This is an alias for the builtin :func:`open` function. + .. audit-event:: open "path mode flags" + + This function raises an :func:`auditing event ` ``open`` with + arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags`` + arguments may have been modified or inferred from the original call. + + +.. function:: open_code(path) + + Opens the provided file with mode ``'rb'``. This function should be used + when the intent is to treat the contents as executable code. + + ``path`` should be an absolute path. + + The behavior of this function may be overridden by an earlier call to the + :c:func:`PyFile_SetOpenCodeHook`, however, it should always be considered + interchangeable with ``open(path, 'rb')``. Overriding the behavior is + intended for additional validation or preprocessing of the file. + + .. versionadded:: 3.8 + .. exception:: BlockingIOError diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 49f932d03845c8..ff937d27c6ce79 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -36,6 +36,24 @@ Number-theoretic and representation functions :class:`~numbers.Integral` value. +.. function:: comb(n, k) + + Return the number of ways to choose *k* items from *n* items without repetition + and without order. + + Evaluates to ``n! / (k! * (n - k)!)`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + Also called the binomial coefficient because it is equivalent + to the coefficient of k-th term in polynomial expansion of the + expression ``(1 + x) ** n``. + + Raises :exc:`TypeError` if either of the arguments are not integers. + Raises :exc:`ValueError` if either of the arguments are negative. + + .. versionadded:: 3.8 + + .. function:: copysign(x, y) Return a float with the magnitude (absolute value) of *x* but the sign of @@ -50,7 +68,7 @@ Number-theoretic and representation functions .. function:: factorial(x) - Return *x* factorial. Raises :exc:`ValueError` if *x* is not integral or + Return *x* factorial as an integer. Raises :exc:`ValueError` if *x* is not integral or is negative. @@ -166,6 +184,20 @@ Number-theoretic and representation functions Return ``True`` if *x* is a NaN (not a number), and ``False`` otherwise. +.. function:: isqrt(n) + + Return the integer square root of the nonnegative integer *n*. This is the + floor of the exact square root of *n*, or equivalently the greatest integer + *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. + + For some applications, it may be more convenient to have the least integer + *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of + the exact square root of *n*. For positive *n*, this can be computed using + ``a = 1 + isqrt(n - 1)``. + + .. versionadded:: 3.8 + + .. function:: ldexp(x, i) Return ``x * (2**i)``. This is essentially the inverse of function @@ -178,6 +210,23 @@ Number-theoretic and representation functions of *x* and are floats. +.. function:: perm(n, k=None) + + Return the number of ways to choose *k* items from *n* items + without repetition and with order. + + Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + If *k* is not specified or is None, then *k* defaults to *n* + and the function returns ``n!``. + + Raises :exc:`TypeError` if either of the arguments are not integers. + Raises :exc:`ValueError` if either of the arguments are negative. + + .. versionadded:: 3.8 + + .. function:: prod(iterable, *, start=1) Calculate the product of all the elements in the input *iterable*. @@ -538,3 +587,6 @@ Constants Module :mod:`cmath` Complex number versions of many of these functions. + +.. |nbsp| unicode:: 0xA0 + :trim: diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 0f895d76b83fa8..c7a13abad888d9 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -67,6 +67,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length will be relative to the offset from the beginning of the file. *offset* defaults to 0. *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`. + .. audit-event:: mmap.__new__ "fileno length access offset" .. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset]) :noindex: @@ -155,6 +156,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length mm.close() + .. audit-event:: mmap.__new__ "fileno length access offset" Memory-mapped file objects support the following methods: @@ -201,6 +203,20 @@ To map anonymous memory, -1 should be passed as the fileno along with the length exception was raised on error under Unix. + .. method:: madvise(option[, start[, length]]) + + Send advice *option* to the kernel about the memory region beginning at + *start* and extending *length* bytes. *option* must be one of the + :ref:`MADV_* constants ` available on the system. If + *start* and *length* are omitted, the entire mapping is spanned. On + some systems (including Linux), *start* must be a multiple of the + :const:`PAGESIZE`. + + Availability: Systems with the ``madvise()`` system call. + + .. versionadded:: 3.8 + + .. method:: move(dest, src, count) Copy the *count* bytes starting at offset *src* to the destination index @@ -290,3 +306,38 @@ To map anonymous memory, -1 should be passed as the fileno along with the length position of the file pointer; the file position is advanced by ``1``. If the mmap was created with :const:`ACCESS_READ`, then writing to it will raise a :exc:`TypeError` exception. + +.. _madvise-constants: + +MADV_* Constants +++++++++++++++++ + +.. data:: MADV_NORMAL + MADV_RANDOM + MADV_SEQUENTIAL + MADV_WILLNEED + MADV_DONTNEED + MADV_REMOVE + MADV_DONTFORK + MADV_DOFORK + MADV_HWPOISON + MADV_MERGEABLE + MADV_UNMERGEABLE + MADV_SOFT_OFFLINE + MADV_HUGEPAGE + MADV_NOHUGEPAGE + MADV_DONTDUMP + MADV_DODUMP + MADV_FREE + MADV_NOSYNC + MADV_AUTOSYNC + MADV_NOCORE + MADV_CORE + MADV_PROTECT + + These options can be passed to :meth:`mmap.madvise`. Not every option will + be present on every system. + + Availability: Systems with the madvise() system call. + + .. versionadded:: 3.8 diff --git a/Doc/library/modules.rst b/Doc/library/modules.rst index 6b2a40a1b71476..565ce0525c2c99 100644 --- a/Doc/library/modules.rst +++ b/Doc/library/modules.rst @@ -17,3 +17,4 @@ The full list of modules described in this chapter is: modulefinder.rst runpy.rst importlib.rst + importlib.metadata.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index c6ffb00819c32c..d8182feab963a1 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -102,7 +102,7 @@ to start a process. These *start methods* are will not be inherited. Starting a process using this method is rather slow compared to using *fork* or *forkserver*. - Available on Unix and Windows. The default on Windows. + Available on Unix and Windows. The default on Windows and macOS. *fork* The parent process uses :func:`os.fork` to fork the Python @@ -124,6 +124,12 @@ to start a process. These *start methods* are Available on Unix platforms which support passing file descriptors over Unix pipes. +.. versionchanged:: 3.8 + + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess. See :issue:`33725`. + .. versionchanged:: 3.4 *spawn* added on all unix platforms, and *forkserver* added for some unix platforms. @@ -944,6 +950,14 @@ Miscellaneous An analogue of :func:`threading.current_thread`. +.. function:: parent_process() + + Return the :class:`Process` object corresponding to the parent process of + the :func:`current_process`. For the main process, ``parent_process`` will + be ``None``. + + .. versionadded:: 3.8 + .. function:: freeze_support() Add support for when a program which uses :mod:`multiprocessing` has been @@ -2266,6 +2280,10 @@ with the :class:`Pool` class. Return whether the call completed without raising an exception. Will raise :exc:`AssertionError` if the result is not ready. + .. versionchanged:: 3.7 + If the result is not ready, :exc:`ValueError` is raised instead of + :exc:`AssertionError`. + The following example demonstrates the use of a pool:: from multiprocessing import Pool diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 5d0ea7dfdd8928..fa02bde84650e1 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -339,7 +339,7 @@ expect a function argument. [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)] -.. function:: methodcaller(name[, args...]) +.. function:: methodcaller(name, /, *args, **kwargs) Return a callable object that calls the method *name* on its operand. If additional arguments and/or keyword arguments are given, they will be given @@ -352,7 +352,7 @@ expect a function argument. Equivalent to:: - def methodcaller(name, *args, **kwargs): + def methodcaller(name, /, *args, **kwargs): def caller(obj): return getattr(obj, name)(*args, **kwargs) return caller diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 8e7ee8bfe7845f..a673b81278ea74 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -87,9 +87,10 @@ the :mod:`glob` module.) .. function:: commonpath(paths) Return the longest common sub-path of each pathname in the sequence - *paths*. Raise :exc:`ValueError` if *paths* contains both absolute and relative - pathnames, or if *paths* is empty. Unlike :func:`commonprefix`, this - returns a valid path. + *paths*. Raise :exc:`ValueError` if *paths* contain both absolute + and relative pathnames, the *paths* are on the different drives or + if *paths* is empty. Unlike :func:`commonprefix`, this returns a + valid path. .. availability:: Unix, Windows. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index e77a8fed377ad6..f0df35e9ddd540 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -32,6 +32,7 @@ Notes on the availability of these functions: objects, and result in an object of the same type, if a path or file name is returned. +* On VxWorks, os.fork, os.execv and os.spawn*p* are not supported. .. note:: @@ -650,7 +651,7 @@ process and user. File Object Creation -------------------- -This function creates new :term:`file objects `. (See also +These functions create new :term:`file objects `. (See also :func:`~os.open` for opening file descriptors.) @@ -706,6 +707,28 @@ as internal buffering of data. pass +.. function:: copy_file_range(src, dst, count, offset_src=None, offset_dst=None) + + Copy *count* bytes from file descriptor *src*, starting from offset + *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. + If *offset_src* is None, then *src* is read from the current position; + respectively for *offset_dst*. The files pointed by *src* and *dst* + must reside in the same filesystem, otherwise an :exc:`OSError` is + raised with :attr:`~OSError.errno` set to :data:`errno.EXDEV`. + + This copy is done without the additional cost of transferring data + from the kernel to user space and then back into the kernel. Additionally, + some filesystems could implement extra optimizations. The copy is done as if + both files are opened as binary. + + The return value is the amount of bytes copied. This could be less than the + amount requested. + + .. availability:: Linux kernel >= 4.5 or glibc >= 2.27. + + .. versionadded:: 3.8 + + .. function:: device_encoding(fd) Return a string describing the encoding of the device associated with *fd* @@ -828,11 +851,14 @@ as internal buffering of data. most *length* bytes in size. As of Python 3.3, this is equivalent to ``os.truncate(fd, length)``. + .. audit-event:: os.truncate "fd length" + .. availability:: Unix, Windows. .. versionchanged:: 3.5 Added support for Windows + .. function:: get_blocking(fd) Get the blocking mode of the file descriptor: ``False`` if the @@ -844,6 +870,7 @@ as internal buffering of data. .. versionadded:: 3.5 + .. function:: isatty(fd) Return ``True`` if the file descriptor *fd* is open and connected to a @@ -911,6 +938,8 @@ as internal buffering of data. This function can support :ref:`paths relative to directory descriptors ` with the *dir_fd* parameter. + .. audit-event:: open "path mode flags" + .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. @@ -2755,6 +2784,8 @@ features: This function can support :ref:`specifying a file descriptor `. + .. audit-event:: os.truncate "path length" + .. availability:: Unix, Windows. .. versionadded:: 3.3 @@ -2973,6 +3004,51 @@ features: Added support for :class:`bytes` paths. +.. function:: memfd_create(name[, flags=os.MFD_CLOEXEC]) + + Create an anonymous file and return a file descriptor that refers to it. + *flags* must be one of the ``os.MFD_*`` constants available on the system + (or a bitwise ORed combination of them). By default, the new file + descriptor is :ref:`non-inheritable `. + + The name supplied in *name* is used as a filename and will be displayed as + the target of the corresponding symbolic link in the directory + ``/proc/self/fd/``. The displayed name is always prefixed with ``memfd:`` + and serves only for debugging purposes. Names do not affect the behavior of + the file descriptor, and as such multiple files can have the same name + without any side effects. + + .. availability:: Linux 3.17 or newer with glibc 2.27 or newer. + + .. versionadded:: 3.8 + + +.. data:: MFD_CLOEXEC + MFD_ALLOW_SEALING + MFD_HUGETLB + MFD_HUGE_SHIFT + MFD_HUGE_MASK + MFD_HUGE_64KB + MFD_HUGE_512KB + MFD_HUGE_1MB + MFD_HUGE_2MB + MFD_HUGE_8MB + MFD_HUGE_16MB + MFD_HUGE_32MB + MFD_HUGE_256MB + MFD_HUGE_512MB + MFD_HUGE_1GB + MFD_HUGE_2GB + MFD_HUGE_16GB + + These flags can be passed to :func:`memfd_create`. + + .. availability:: Linux 3.17 or newer with glibc 2.27 or newer. The + ``MFD_HUGE*`` flags are only available since Linux 4.14. + + .. versionadded:: 3.8 + + Linux extended attributes ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -3578,6 +3654,9 @@ written in Python, such as a mail server's external command delivery program. process. On Windows, the process id will actually be the process handle, so can be used with the :func:`waitpid` function. + Note on VxWorks, this function doesn't return ``-signal`` when the new process is + killed. Instead it raises OSError exception. + The "l" and "v" variants of the :func:`spawn\* ` functions differ in how command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the @@ -3711,6 +3790,8 @@ written in Python, such as a mail server's external command delivery program. to using this function. See the :ref:`subprocess-replacements` section in the :mod:`subprocess` documentation for some helpful recipes. + .. audit-event:: os.system command + .. availability:: Unix, Windows. diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 27721e698826a1..6aa30492c7060c 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -195,34 +195,29 @@ The :mod:`pickle` module provides the following constants: The :mod:`pickle` module provides the following functions to make the pickling process more convenient: -.. function:: dump(obj, file, protocol=None, \*, fix_imports=True) +.. function:: dump(obj, file, protocol=None, \*, fix_imports=True, buffer_callback=None) Write a pickled representation of *obj* to the open :term:`file object` *file*. This is equivalent to ``Pickler(file, protocol).dump(obj)``. - The optional *protocol* argument, an integer, tells the pickler to use - the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`. - If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative - number is specified, :data:`HIGHEST_PROTOCOL` is selected. + Arguments *file*, *protocol*, *fix_imports* and *buffer_callback* have + the same meaning as in the :class:`Pickler` constructor. - The *file* argument must have a write() method that accepts a single bytes - argument. It can thus be an on-disk file opened for binary writing, an - :class:`io.BytesIO` instance, or any other custom object that meets this - interface. - - If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3 names to the old module names used in Python 2, so - that the pickle data stream is readable with Python 2. + .. versionchanged:: 3.8 + The *buffer_callback* argument was added. -.. function:: dumps(obj, protocol=None, \*, fix_imports=True) +.. function:: dumps(obj, protocol=None, \*, fix_imports=True, buffer_callback=None) Return the pickled representation of the object as a :class:`bytes` object, instead of writing it to a file. - Arguments *protocol* and *fix_imports* have the same meaning as in - :func:`dump`. + Arguments *protocol*, *fix_imports* and *buffer_callback* have the same + meaning as in the :class:`Pickler` constructor. + + .. versionchanged:: 3.8 + The *buffer_callback* argument was added. -.. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict") +.. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) Read a pickled object representation from the open :term:`file object` *file* and return the reconstituted object hierarchy specified therein. @@ -232,24 +227,13 @@ process more convenient: protocol argument is needed. Bytes past the pickled object's representation are ignored. - The argument *file* must have two methods, a read() method that takes an - integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file opened for - binary reading, an :class:`io.BytesIO` object, or any other custom object - that meets this interface. - - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. - Using ``encoding='latin1'`` is required for unpickling NumPy arrays and - instances of :class:`~datetime.datetime`, :class:`~datetime.date` and - :class:`~datetime.time` pickled by Python 2. + Arguments *file*, *fix_imports*, *encoding*, *errors*, *strict* and *buffers* + have the same meaning as in the :class:`Unpickler` constructor. -.. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict") + .. versionchanged:: 3.8 + The *buffers* argument was added. + +.. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) Read a pickled object hierarchy from a :class:`bytes` object and return the reconstituted object hierarchy specified therein. @@ -258,16 +242,11 @@ process more convenient: protocol argument is needed. Bytes past the pickled object's representation are ignored. - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. - Using ``encoding='latin1'`` is required for unpickling NumPy arrays and - instances of :class:`~datetime.datetime`, :class:`~datetime.date` and - :class:`~datetime.time` pickled by Python 2. + Arguments *file*, *fix_imports*, *encoding*, *errors*, *strict* and *buffers* + have the same meaning as in the :class:`Unpickler` constructor. + + .. versionchanged:: 3.8 + The *buffers* argument was added. The :mod:`pickle` module defines three exceptions: @@ -295,10 +274,10 @@ The :mod:`pickle` module defines three exceptions: IndexError. -The :mod:`pickle` module exports two classes, :class:`Pickler` and -:class:`Unpickler`: +The :mod:`pickle` module exports three classes, :class:`Pickler`, +:class:`Unpickler` and :class:`PickleBuffer`: -.. class:: Pickler(file, protocol=None, \*, fix_imports=True) +.. class:: Pickler(file, protocol=None, \*, fix_imports=True, buffer_callback=None) This takes a binary file for writing a pickle data stream. @@ -316,6 +295,20 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and map the new Python 3 names to the old module names used in Python 2, so that the pickle data stream is readable with Python 2. + If *buffer_callback* is None (the default), buffer views are + serialized into *file* as part of the pickle stream. + + If *buffer_callback* is not None, then it can be called any number + of times with a buffer view. If the callback returns a false value + (such as None), the given buffer is :ref:`out-of-band `; + otherwise the buffer is serialized in-band, i.e. inside the pickle stream. + + It is an error if *buffer_callback* is not None and *protocol* is + None or smaller than 5. + + .. versionchanged:: 3.8 + The *buffer_callback* argument was added. + .. method:: dump(obj) Write a pickled representation of *obj* to the open file object given in @@ -379,26 +372,43 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and Use :func:`pickletools.optimize` if you need more compact pickles. -.. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict") +.. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) This takes a binary file for reading a pickle data stream. The protocol version of the pickle is detected automatically, so no protocol argument is needed. - The argument *file* must have two methods, a read() method that takes an - integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file object + The argument *file* must have three methods, a read() method that takes an + integer argument, a readinto() method that takes a buffer argument + and a readline() method that requires no arguments, as in the + :class:`io.BufferedIOBase` interface. Thus *file* can be an on-disk file opened for binary reading, an :class:`io.BytesIO` object, or any other custom object that meets this interface. - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can + The optional arguments *fix_imports*, *encoding* and *errors* are used + to control compatibility support for pickle stream generated by Python 2. + If *fix_imports* is true, pickle will try to map the old Python 2 names + to the new names used in Python 3. The *encoding* and *errors* tell + pickle how to decode 8-bit string instances pickled by Python 2; + these default to 'ASCII' and 'strict', respectively. The *encoding* can be 'bytes' to read these 8-bit string instances as bytes objects. + Using ``encoding='latin1'`` is required for unpickling NumPy arrays and + instances of :class:`~datetime.datetime`, :class:`~datetime.date` and + :class:`~datetime.time` pickled by Python 2. + + If *buffers* is None (the default), then all data necessary for + deserialization must be contained in the pickle stream. This means + that the *buffer_callback* argument was None when a :class:`Pickler` + was instantiated (or when :func:`dump` or :func:`dumps` was called). + + If *buffers* is not None, it should be an iterable of buffer-enabled + objects that is consumed each time the pickle stream references + an :ref:`out-of-band ` buffer view. Such buffers have been + given in order to the *buffer_callback* of a Pickler object. + + .. versionchanged:: 3.8 + The *buffers* argument was added. .. method:: load() @@ -427,6 +437,35 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and how they can be loaded, potentially reducing security risks. Refer to :ref:`pickle-restrict` for details. + .. audit-event:: pickle.find_class "module name" + +.. class:: PickleBuffer(buffer) + + A wrapper for a buffer representing picklable data. *buffer* must be a + :ref:`buffer-providing ` object, such as a + :term:`bytes-like object` or a N-dimensional array. + + :class:`PickleBuffer` is itself a buffer provider, therefore it is + possible to pass it to other APIs expecting a buffer-providing object, + such as :class:`memoryview`. + + :class:`PickleBuffer` objects can only be serialized using pickle + protocol 5 or higher. They are eligible for + :ref:`out-of-band serialization `. + + .. versionadded:: 3.8 + + .. method:: raw() + + Return a :class:`memoryview` of the memory area underlying this buffer. + The returned object is a one-dimensional, C-contiguous memoryview + with format ``B`` (unsigned bytes). :exc:`BufferError` is raised if + the buffer is neither C- nor Fortran-contiguous. + + .. method:: release() + + Release the underlying buffer exposed by the PickleBuffer object. + .. _pickle-picklable: @@ -863,6 +902,125 @@ a given class:: assert unpickled_class.my_attribute == 1 +.. _pickle-oob: + +Out-of-band Buffers +------------------- + +.. versionadded:: 3.8 + +In some contexts, the :mod:`pickle` module is used to transfer massive amounts +of data. Therefore, it can be important to minimize the number of memory +copies, to preserve performance and resource consumption. However, normal +operation of the :mod:`pickle` module, as it transforms a graph-like structure +of objects into a sequential stream of bytes, intrinsically involves copying +data to and from the pickle stream. + +This constraint can be eschewed if both the *provider* (the implementation +of the object types to be transferred) and the *consumer* (the implementation +of the communications system) support the out-of-band transfer facilities +provided by pickle protocol 5 and higher. + +Provider API +^^^^^^^^^^^^ + +The large data objects to be pickled must implement a :meth:`__reduce_ex__` +method specialized for protocol 5 and higher, which returns a +:class:`PickleBuffer` instance (instead of e.g. a :class:`bytes` object) +for any large data. + +A :class:`PickleBuffer` object *signals* that the underlying buffer is +eligible for out-of-band data transfer. Those objects remain compatible +with normal usage of the :mod:`pickle` module. However, consumers can also +opt-in to tell :mod:`pickle` that they will handle those buffers by +themselves. + +Consumer API +^^^^^^^^^^^^ + +A communications system can enable custom handling of the :class:`PickleBuffer` +objects generated when serializing an object graph. + +On the sending side, it needs to pass a *buffer_callback* argument to +:class:`Pickler` (or to the :func:`dump` or :func:`dumps` function), which +will be called with each :class:`PickleBuffer` generated while pickling +the object graph. Buffers accumulated by the *buffer_callback* will not +see their data copied into the pickle stream, only a cheap marker will be +inserted. + +On the receiving side, it needs to pass a *buffers* argument to +:class:`Unpickler` (or to the :func:`load` or :func:`loads` function), +which is an iterable of the buffers which were passed to *buffer_callback*. +That iterable should produce buffers in the same order as they were passed +to *buffer_callback*. Those buffers will provide the data expected by the +reconstructors of the objects whose pickling produced the original +:class:`PickleBuffer` objects. + +Between the sending side and the receiving side, the communications system +is free to implement its own transfer mechanism for out-of-band buffers. +Potential optimizations include the use of shared memory or datatype-dependent +compression. + +Example +^^^^^^^ + +Here is a trivial example where we implement a :class:`bytearray` subclass +able to participate in out-of-band buffer pickling:: + + class ZeroCopyByteArray(bytearray): + + def __reduce_ex__(self, protocol): + if protocol >= 5: + return type(self)._reconstruct, (PickleBuffer(self),), None + else: + # PickleBuffer is forbidden with pickle protocols <= 4. + return type(self)._reconstruct, (bytearray(self),) + + @classmethod + def _reconstruct(cls, obj): + with memoryview(obj) as m: + # Get a handle over the original buffer object + obj = m.obj + if type(obj) is cls: + # Original buffer object is a ZeroCopyByteArray, return it + # as-is. + return obj + else: + return cls(obj) + +The reconstructor (the ``_reconstruct`` class method) returns the buffer's +providing object if it has the right type. This is an easy way to simulate +zero-copy behaviour on this toy example. + +On the consumer side, we can pickle those objects the usual way, which +when unserialized will give us a copy of the original object:: + + b = ZeroCopyByteArray(b"abc") + data = pickle.dumps(b, protocol=5) + new_b = pickle.loads(data) + print(b == new_b) # True + print(b is new_b) # False: a copy was made + +But if we pass a *buffer_callback* and then give back the accumulated +buffers when unserializing, we are able to get back the original object:: + + b = ZeroCopyByteArray(b"abc") + buffers = [] + data = pickle.dumps(b, protocol=5, buffer_callback=buffers.append) + new_b = pickle.loads(data, buffers=buffers) + print(b == new_b) # True + print(b is new_b) # True: no copy was made + +This example is limited by the fact that :class:`bytearray` allocates its +own memory: you cannot create a :class:`bytearray` instance that is backed +by another object's memory. However, third-party datatypes such as NumPy +arrays do not have this limitation, and allow use of zero-copy pickling +(or making as few copies as possible) when transferring between distinct +processes or systems. + +.. seealso:: :pep:`574` -- Pickle protocol 5 with out-of-band data + + .. _pickle-restrict: Restricting Globals diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index d8039fd28e0d8a..8d589d247b7747 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -309,7 +309,7 @@ functions: Profile the cmd via :func:`exec` with the specified global and local environment. - .. method:: runcall(func, *args, **kwargs) + .. method:: runcall(func, /, *args, **kwargs) Profile ``func(*args, **kwargs)`` diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 0ab766065d6e81..12268437d07e98 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -43,11 +43,32 @@ The :mod:`pty` module defines the following functions: Spawn a process, and connect its controlling terminal with the current process's standard io. This is often used to baffle programs which insist on - reading from the controlling terminal. + reading from the controlling terminal. It is expected that the process + spawned behind the pty will eventually terminate, and when it does *spawn* + will return. + + The functions *master_read* and *stdin_read* are passed a file descriptor + which they should read from, and they should always return a byte string. In + order to force spawn to return before the child process exits an + :exc:`OSError` should be thrown. + + The default implementation for both functions will read and return up to 1024 + bytes each time the function is called. The *master_read* callback is passed + the pseudoterminal’s master file descriptor to read output from the child + process, and *stdin_read* is passed file descriptor 0, to read from the + parent process's standard input. + + Returning an empty byte string from either callback is interpreted as an + end-of-file (EOF) condition, and that callback will not be called after + that. If *stdin_read* signals EOF the controlling terminal can no longer + communicate with the parent process OR the child process. Unless the child + process will quit without any input, *spawn* will then loop forever. If + *master_read* signals EOF the same behavior results (on linux at least). + + If both callbacks signal EOF then *spawn* will probably never return, unless + *select* throws an error on your platform when passed three empty lists. This + is a bug, documented in `issue 26228 `_. - The functions *master_read* and *stdin_read* should be functions which read from - a file descriptor. The defaults try to read 1024 bytes each time they are - called. .. versionchanged:: 3.4 :func:`spawn` now returns the status value from :func:`os.waitpid` diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 8cb5a4d546c874..3824353abda1f8 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -42,6 +42,13 @@ byte-code cache files in the directory containing the source code. is raised. This function returns the path to byte-compiled file, i.e. whatever *cfile* value was used. + The *doraise* and *quiet* arguments determine how errors are handled while + compiling file. If *quiet* is 0 or 1, and *doraise* is false, the default + behaviour is enabled: an error string is written to ``sys.stderr``, and the + function returns ``None`` instead of a path. If *doraise* is true, + a :exc:`PyCompileError` is raised instead. However if *quiet* is 2, + no message is written, and *doraise* has no effect. + If the path that *cfile* becomes (either explicitly specified or computed) is a symlink or non-regular file, :exc:`FileExistsError` will be raised. This is to act as a warning that import will turn those paths into regular @@ -82,6 +89,9 @@ byte-code cache files in the directory containing the source code. overrides the value of the *invalidation_mode* argument, and determines its default value instead. + .. versionchanged:: 3.8 + The *quiet* parameter was added. + .. class:: PycInvalidationMode diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 16c28cf7d02fb9..eae0a6df45f30e 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -19,7 +19,7 @@ function. Readline keybindings may be configured via an initialization file, typically ``.inputrc`` in your home directory. See `Readline Init File -`_ +`_ in the GNU Readline manual for information about the format and allowable constructs of that file, and the capabilities of the Readline library in general. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index ad96dbc95b0cd2..fab16f52c46218 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -20,8 +20,7 @@ scheduler: The :class:`scheduler` class defines a generic interface to scheduling events. It needs two functions to actually deal with the "outside world" --- *timefunc* should be callable without arguments, and return a number (the "time", in any - units whatsoever). If time.monotonic is not available, the *timefunc* default - is time.time instead. The *delayfunc* function should be callable with one + units whatsoever). The *delayfunc* function should be callable with one argument, compatible with the output of *timefunc*, and should delay that many time units. *delayfunc* will also be called with the argument ``0`` after each event is run to allow other threads an opportunity to run in multi-threaded diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index fb335c69006816..a8421fdb7008ce 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -37,6 +37,21 @@ The :mod:`shlex` module defines the following functions: standard input. +.. function:: join(split_command) + + Concatenate the tokens of the list *split_command* and return a string. + This function is the inverse of :func:`split`. + + >>> from shlex import join + >>> print(join(['echo', '-n', 'Multiple words'])) + echo -n 'Multiple words' + + The returned value is shell-escaped to protect against injection + vulnerabilities (see :func:`quote`). + + .. versionadded:: 3.8 + + .. function:: quote(s) Return a shell-escaped version of the string *s*. The returned value is a @@ -210,7 +225,8 @@ variables which either control lexical analysis or can be used for debugging: appear in filename specifications and command line parameters, will also be included in this attribute, and any characters which appear in ``punctuation_chars`` will be removed from ``wordchars`` if they are present - there. + there. If :attr:`whitespace_split` is set to ``True``, this will have no + effect. .. attribute:: shlex.whitespace @@ -243,11 +259,13 @@ variables which either control lexical analysis or can be used for debugging: If ``True``, tokens will only be split in whitespaces. This is useful, for example, for parsing command lines with :class:`~shlex.shlex`, getting - tokens in a similar way to shell arguments. If this attribute is ``True``, - :attr:`punctuation_chars` will have no effect, and splitting will happen - only on whitespaces. When using :attr:`punctuation_chars`, which is - intended to provide parsing closer to that implemented by shells, it is - advisable to leave ``whitespace_split`` as ``False`` (the default value). + tokens in a similar way to shell arguments. When used in combination with + :attr:`punctuation_chars`, tokens will be split on whitespace in addition to + those characters. + + .. versionchanged:: 3.8 + The :attr:`punctuation_chars` attribute was made compatible with the + :attr:`whitespace_split` attribute. .. attribute:: shlex.infile @@ -383,12 +401,15 @@ otherwise. To illustrate, you can see the difference in the following snippet: >>> import shlex >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" - >>> list(shlex.shlex(text)) - ['a', '&', '&', 'b', ';', 'c', '&', '&', 'd', '|', '|', 'e', ';', 'f', '>', - "'abc'", ';', '(', 'def', '"ghi"', ')'] - >>> list(shlex.shlex(text, punctuation_chars=True)) - ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', "'abc'", - ';', '(', 'def', '"ghi"', ')'] + >>> s = shlex.shlex(text, posix=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] + >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) + >>> s.whitespace_split = True + >>> list(s) + ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', + '(', 'def', 'ghi', ')'] Of course, tokens will be returned which are not valid for shells, and you'll need to implement your own error checks on the returned tokens. @@ -413,6 +434,11 @@ which characters constitute punctuation. For example:: >>> list(s) ['~/a', '&&', 'b-c', '--color=auto', '||', 'd', '*.py?'] + However, to match the shell as closely as possible, it is recommended to + always use ``posix`` and :attr:`~shlex.whitespace_split` when using + :attr:`~shlex.punctuation_chars`, which will negate + :attr:`~shlex.wordchars` entirely. + For best effect, ``punctuation_chars`` should be set in conjunction with ``posix=True``. (Note that ``posix=False`` is the default for :class:`~shlex.shlex`.) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 4af5a16806088d..dcb2a16cff98cb 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -420,8 +420,7 @@ the use of userspace buffers in Python as in "``outfd.write(infd.read())``". On macOS `fcopyfile`_ is used to copy the file content (not metadata). -On Linux, Solaris and other POSIX platforms where :func:`os.sendfile` supports -copies between 2 regular file descriptors :func:`os.sendfile` is used. +On Linux :func:`os.sendfile` is used. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 64 KiB) and a :func:`memoryview`-based variant of diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 379633a3b6052b..e0dbbb4c0d3201 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -380,7 +380,7 @@ Constants Enables CAN FD support in a CAN_RAW socket. This is disabled by default. This allows your application to send both CAN and CAN FD frames; however, - you one must accept both CAN and CAN FD frames when reading from the socket. + you must accept both CAN and CAN FD frames when reading from the socket. This constant is documented in the Linux documentation. @@ -526,6 +526,8 @@ The following functions all create :ref:`socket objects `. The newly created socket is :ref:`non-inheritable `. + .. audit-event:: socket.__new__ "self family type protocol" + .. versionchanged:: 3.3 The AF_CAN family was added. The AF_RDS family was added. @@ -718,6 +720,8 @@ The :mod:`socket` module also offers various network-related services: :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` method. + .. audit-event:: socket.getaddrinfo "host port family type protocol" + The following example fetches address information for a hypothetical TCP connection to ``example.org`` on port 80 (results may differ on your system if IPv6 isn't enabled):: @@ -753,6 +757,8 @@ The :mod:`socket` module also offers various network-related services: interface. :func:`gethostbyname` does not support IPv6 name resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. + .. audit-event:: socket.gethostbyname hostname + .. function:: gethostbyname_ex(hostname) @@ -765,12 +771,16 @@ The :mod:`socket` module also offers various network-related services: resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. + .. audit-event:: socket.gethostbyname hostname + .. function:: gethostname() Return a string containing the hostname of the machine where the Python interpreter is currently executing. + .. audit-event:: socket.gethostname + Note: :func:`gethostname` doesn't always return the fully qualified domain name; use :func:`getfqdn` for that. @@ -785,6 +795,8 @@ The :mod:`socket` module also offers various network-related services: domain name, use the function :func:`getfqdn`. :func:`gethostbyaddr` supports both IPv4 and IPv6. + .. audit-event:: socket.gethostbyaddr ip_address + .. function:: getnameinfo(sockaddr, flags) @@ -798,6 +810,8 @@ The :mod:`socket` module also offers various network-related services: For more information about *flags* you can consult :manpage:`getnameinfo(3)`. + .. audit-event:: socket.getnameinfo sockaddr + .. function:: getprotobyname(protocolname) Translate an Internet protocol name (for example, ``'icmp'``) to a constant @@ -813,6 +827,8 @@ The :mod:`socket` module also offers various network-related services: service. The optional protocol name, if given, should be ``'tcp'`` or ``'udp'``, otherwise any protocol will match. + .. audit-event:: socket.getservbyname "servicename protocolname" + .. function:: getservbyport(port[, protocolname]) @@ -820,6 +836,8 @@ The :mod:`socket` module also offers various network-related services: service. The optional protocol name, if given, should be ``'tcp'`` or ``'udp'``, otherwise any protocol will match. + .. audit-event:: socket.getservbyport "port protocolname" + .. function:: ntohl(x) @@ -1003,6 +1021,8 @@ The :mod:`socket` module also offers various network-related services: Set the machine's hostname to *name*. This will raise an :exc:`OSError` if you don't have enough rights. + .. audit-event:: socket.sethostname name + .. availability:: Unix. .. versionadded:: 3.3 @@ -1014,10 +1034,13 @@ The :mod:`socket` module also offers various network-related services: (index int, name string) tuples. :exc:`OSError` if the system call fails. - .. availability:: Unix. + .. availability:: Unix, Windows. .. versionadded:: 3.3 + .. versionchanged:: 3.8 + Windows support was added. + .. function:: if_nametoindex(if_name) @@ -1025,10 +1048,13 @@ The :mod:`socket` module also offers various network-related services: interface name. :exc:`OSError` if no interface with the given name exists. - .. availability:: Unix. + .. availability:: Unix, Windows. .. versionadded:: 3.3 + .. versionchanged:: 3.8 + Windows support was added. + .. function:: if_indextoname(if_index) @@ -1036,10 +1062,13 @@ The :mod:`socket` module also offers various network-related services: interface index number. :exc:`OSError` if no interface with the given index exists. - .. availability:: Unix. + .. availability:: Unix, Windows. .. versionadded:: 3.3 + .. versionchanged:: 3.8 + Windows support was added. + .. _socket-objects: @@ -1078,6 +1107,7 @@ to sockets. Bind the socket to *address*. The socket must not already be bound. (The format of *address* depends on the address family --- see above.) + .. audit-event:: socket.bind "self address" .. method:: socket.close() @@ -1115,6 +1145,8 @@ to sockets. :exc:`InterruptedError` exception if the connection is interrupted by a signal (or the exception raised by the signal handler). + .. audit-event:: socket.connect "self address" + .. versionchanged:: 3.5 The method now waits until the connection completes instead of raising an :exc:`InterruptedError` exception if the connection is interrupted by a @@ -1131,6 +1163,7 @@ to sockets. :c:data:`errno` variable. This is useful to support, for example, asynchronous connects. + .. audit-event:: socket.connect "self address" .. method:: socket.detach() @@ -1472,6 +1505,8 @@ to sockets. bytes sent. (The format of *address* depends on the address family --- see above.) + .. audit-event:: socket.sendto "self address" + .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an exception, the method now retries the system call instead of raising @@ -1511,6 +1546,8 @@ to sockets. .. availability:: most Unix platforms, possibly others. + .. audit-event:: socket.sendmsg "self address" + .. versionadded:: 3.3 .. versionchanged:: 3.5 diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 37087ac5af492a..20fca54aab144f 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -537,6 +537,7 @@ Connection Objects with open('dump.sql', 'w') as f: for line in con.iterdump(): f.write('%s\n' % line) + con.close() .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) @@ -573,8 +574,11 @@ Connection Objects print(f'Copied {total-remaining} of {total} pages...') con = sqlite3.connect('existing_db.db') - with sqlite3.connect('backup.db') as bck: + bck = sqlite3.connect('backup.db') + with bck: con.backup(bck, pages=1, progress=progress) + bck.close() + con.close() Example 2, copy an existing database into a transient copy:: diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 20f5724447164d..279af5728913d9 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -139,6 +139,10 @@ purposes. *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load default CA certificates. + When :attr:`~SSLContext.keylog_filename` is supported and the environment + variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` + enables key logging. + .. note:: The protocol, options, cipher and other settings may change to more restrictive values anytime without prior deprecation. The values @@ -172,6 +176,10 @@ purposes. 3DES was dropped from the default cipher string. + .. versionchanged:: 3.8 + + Support for key logging to :envvar:`SSLKEYLOGFILE` was added. + Exceptions ^^^^^^^^^^ @@ -1056,6 +1064,7 @@ Constants SSL 3.0 to TLS 1.3. + SSL Sockets ----------- @@ -1901,6 +1910,20 @@ to speed up repeated connections from the same clients. This features requires OpenSSL 0.9.8f or newer. +.. attribute:: SSLContext.keylog_filename + + Write TLS keys to a keylog file, whenever key material is generated or + received. The keylog file is designed for debugging purposes only. The + file format is specified by NSS and used by many traffic analyzers such + as Wireshark. The log file is opened in append-only mode. Writes are + synchronized between threads, but not between processes. + + .. versionadded:: 3.8 + + .. note:: + + This features requires OpenSSL 1.1.1 or newer. + .. attribute:: SSLContext.maximum_version A :class:`TLSVersion` enum member representing the highest supported @@ -1936,6 +1959,19 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.7 +.. attribute:: SSLContext.num_tickets + + Control the number of TLS 1.3 session tickets of a + :attr:`TLS_PROTOCOL_SERVER` context. The setting has no impact on TLS + 1.0 to 1.2 connections. + + .. note:: + + This attribute is not available unless the ssl module is compiled + with OpenSSL 1.1.1 or newer. + + .. versionadded:: 3.8 + .. attribute:: SSLContext.options An integer representing the set of SSL options enabled on this context. diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index fb7df4e7188a07..bc841fda72f887 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -511,22 +511,33 @@ However, for reading convenience, most of the examples show sorted sequences. is not least 1. The *dist* can be any iterable containing sample data or it can be an - instance of a class that defines an :meth:`~inv_cdf` method. + instance of a class that defines an :meth:`~inv_cdf` method. For meaningful + results, the number of data points in *dist* should be larger than *n*. Raises :exc:`StatisticsError` if there are not at least two data points. For sample data, the cut points are linearly interpolated from the two nearest data points. For example, if a cut point falls one-third of the distance between two sample values, ``100`` and ``112``, the - cut-point will evaluate to ``104``. Other selection methods may be - offered in the future (for example choose ``100`` as the nearest - value or compute ``106`` as the midpoint). This might matter if - there are too few samples for a given number of cut points. - - If *method* is set to *inclusive*, *dist* is treated as population data. - The minimum value is treated as the 0th percentile and the maximum - value is treated as the 100th percentile. If *dist* is an instance of - a class that defines an :meth:`~inv_cdf` method, setting *method* - has no effect. + cut-point will evaluate to ``104``. + + The *method* for computing quantiles can be varied depending on + whether the data in *dist* includes or excludes the lowest and + highest possible values from the population. + + The default *method* is "exclusive" and is used for data sampled from + a population that can have more extreme values than found in the + samples. The portion of the population falling below the *i-th* of + *m* data points is computed as ``i / (m + 1)``. + + Setting the *method* to "inclusive" is used for describing population + data or for samples that include the extreme points. The minimum + value in *dist* is treated as the 0th percentile and the maximum + value is treated as the 100th percentile. The portion of the + population falling below the *i-th* of *m* data points is computed as + ``(i - 1) / (m - 1)``. + + If *dist* is an instance of a class that defines an + :meth:`~inv_cdf` method, setting *method* has no effect. .. doctest:: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 53337291dd39ce..35a17a18080996 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1114,7 +1114,7 @@ Notes: item is removed and returned. (3) - ``remove`` raises :exc:`ValueError` when *x* is not found in *s*. + :meth:`remove` raises :exc:`ValueError` when *x* is not found in *s*. (4) The :meth:`reverse` method modifies the sequence in place for economy of @@ -1124,7 +1124,9 @@ Notes: (5) :meth:`clear` and :meth:`!copy` are included for consistency with the interfaces of mutable containers that don't support slicing operations - (such as :class:`dict` and :class:`set`) + (such as :class:`dict` and :class:`set`). :meth:`!copy` is not part of the + :class:`collections.abc.MutableSequence` ABC, but most concrete + mutable sequence classes provide it. .. versionadded:: 3.3 :meth:`clear` and :meth:`!copy` methods. @@ -2402,8 +2404,26 @@ data and are closely related to string objects in a variety of other ways. >>> b'\xf0\xf1\xf2'.hex() 'f0f1f2' + If you want to make the hex string easier to read, you can specify a + single character separator *sep* parameter to include in the output. + By default between each byte. A second optional *bytes_per_sep* + parameter controls the spacing. Positive values calculate the + separator position from the right, negative values from the left. + + >>> value = b'\xf0\xf1\xf2' + >>> value.hex('-') + 'f0-f1-f2' + >>> value.hex('_', 2) + 'f0_f1f2' + >>> b'UUDDLRLRAB'.hex(' ', -4) + '55554444 4c524c52 4142' + .. versionadded:: 3.5 + .. versionchanged:: 3.8 + :meth:`bytes.hex` now supports optional *sep* and *bytes_per_sep* + parameters to insert separators between bytes in the hex output. + Since bytes objects are sequences of integers (akin to a tuple), for a bytes object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes object of length 1. (This contrasts with text strings, where both indexing @@ -2716,8 +2736,8 @@ arbitrary binary data. The prefix(es) to search for may be any :term:`bytes-like object`. -.. method:: bytes.translate(table, delete=b'') - bytearray.translate(table, delete=b'') +.. method:: bytes.translate(table, /, delete=b'') + bytearray.translate(table, /, delete=b'') Return a copy of the bytes or bytearray object where all bytes occurring in the optional argument *delete* are removed, and the remaining bytes have diff --git a/Doc/library/string.rst b/Doc/library/string.rst index c2f65224bc8daf..288dde6b3fe4c0 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -88,7 +88,7 @@ implementation as the built-in :meth:`~str.format` method. The :class:`Formatter` class has the following public methods: - .. method:: format(format_string, *args, **kwargs) + .. method:: format(format_string, /, *args, **kwargs) The primary API method. It takes a format string and an arbitrary set of positional and keyword arguments. @@ -720,7 +720,7 @@ these rules. The methods of :class:`Template` are: The constructor takes a single argument which is the template string. - .. method:: substitute(mapping, **kwds) + .. method:: substitute(mapping={}, /, **kwds) Performs the template substitution, returning a new string. *mapping* is any dictionary-like object with keys that match the placeholders in the @@ -729,7 +729,7 @@ these rules. The methods of :class:`Template` are: and there are duplicates, the placeholders from *kwds* take precedence. - .. method:: safe_substitute(mapping, **kwds) + .. method:: safe_substitute(mapping={}, /, **kwds) Like :meth:`substitute`, except that if placeholders are missing from *mapping* and *kwds*, instead of raising a :exc:`KeyError` exception, the diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 1a0fd73c6758c7..a06d90344ba34e 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -70,7 +70,7 @@ The module defines the following exception and functions: size required by the format, as reflected by :func:`calcsize`. -.. function:: unpack_from(format, buffer, offset=0) +.. function:: unpack_from(format, /, buffer, offset=0) Unpack from *buffer* starting at position *offset*, according to the format string *format*. The result is a tuple even if it contains exactly one diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index d840b461f98cb7..ede5c3c5a369e3 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -347,7 +347,8 @@ functions. the class uses the Windows ``CreateProcess()`` function. The arguments to :class:`Popen` are as follows. - *args* should be a sequence of program arguments or else a single string. + *args* should be a sequence of program arguments or else a single string + or :term:`path-like object`. By default, the program to execute is the first item in *args* if *args* is a sequence. If *args* is a string, the interpretation is platform-dependent and described below. See the *shell* and *executable* @@ -381,6 +382,15 @@ functions. manner described in :ref:`converting-argument-sequence`. This is because the underlying ``CreateProcess()`` operates on strings. + .. versionchanged:: 3.6 + *args* parameter accepts a :term:`path-like object` if *shell* is + ``False`` and a sequence containing path-like objects on POSIX. + + .. versionchanged:: 3.8 + *args* parameter accepts a :term:`path-like object` if *shell* is + ``False`` and a sequence containing bytes and path-like objects + on Windows. + The *shell* argument (which defaults to ``False``) specifies whether to use the shell as the program to execute. If *shell* is ``True``, it is recommended to pass *args* as a string rather than as a sequence. @@ -436,6 +446,13 @@ functions. :program:`ps`. If ``shell=True``, on POSIX the *executable* argument specifies a replacement shell for the default :file:`/bin/sh`. + .. versionchanged:: 3.6 + *executable* parameter accepts a :term:`path-like object` on POSIX. + + .. versionchanged:: 3.8 + *executable* parameter accepts a bytes and :term:`path-like object` + on Windows. + *stdin*, *stdout* and *stderr* specify the executed program's standard input, standard output and standard error file handles, respectively. Valid values are :data:`PIPE`, :data:`DEVNULL`, an existing file descriptor (a positive @@ -492,13 +509,19 @@ functions. The *pass_fds* parameter was added. If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. *cwd* can be a :class:`str` and + *cwd* before executing the child. *cwd* can be a string, bytes or :term:`path-like ` object. In particular, the function looks for *executable* (or for the first item in *args*) relative to *cwd* if the executable path is a relative path. .. versionchanged:: 3.6 - *cwd* parameter accepts a :term:`path-like object`. + *cwd* parameter accepts a :term:`path-like object` on POSIX. + + .. versionchanged:: 3.7 + *cwd* parameter accepts a :term:`path-like object` on Windows. + + .. versionchanged:: 3.8 + *cwd* parameter accepts a bytes object on Windows. If *restore_signals* is true (the default) all signals that Python has set to SIG_IGN are restored to SIG_DFL in the child process before the exec. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 5039ffa933ac54..c073431c894817 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -16,9 +16,37 @@ always available. On POSIX systems where Python was built with the standard ``configure`` script, this contains the ABI flags as specified by :pep:`3149`. + .. versionchanged:: 3.8 + Default flags became an empty string (``m`` flag for pymalloc has been + removed). + .. versionadded:: 3.2 +.. function:: addaudithook(hook) + + Adds the callable *hook* to the collection of active auditing hooks for the + current interpreter. + + When an auditing event is raised through the :func:`sys.audit` function, each + hook will be called in the order it was added with the event name and the + tuple of arguments. Native hooks added by :c:func:`PySys_AddAuditHook` are + called first, followed by hooks added in the current interpreter. + + Calling this function will trigger an event for all existing hooks, and if + any raise an exception derived from :class:`Exception`, the add will be + silently ignored. As a result, callers cannot assume that their hook has been + added unless they control all existing hooks. + + .. versionadded:: 3.8 + + .. impl-detail:: + + When tracing is enabled, Python hooks are only traced if the callable has + a ``__cantrace__`` member that is set to a true value. Otherwise, trace + functions will not see the hook. + + .. data:: argv The list of command line arguments passed to a Python script. ``argv[0]`` is the @@ -37,6 +65,30 @@ always available. ``[os.fsencode(arg) for arg in sys.argv]``. +.. _auditing: + +.. function:: audit(event, *args) + + .. index:: single: auditing + + Raises an auditing event with any active hooks. The event name is a string + identifying the event and its associated schema, which is the number and + types of arguments. The schema for a given event is considered public and + stable API and should not be modified between releases. + + This function will raise the first exception raised by any hook. In general, + these errors should not be handled and should terminate the process as + quickly as possible. + + Hooks are added using the :func:`sys.addaudithook` or + :c:func:`PySys_AddAuditHook` functions. + + The native equivalent of this function is :c:func:`PySys_Audit`. Using the + native function is preferred when possible. + + .. versionadded:: 3.8 + + .. data:: base_exec_prefix Set during Python startup, before ``site.py`` is run, to the same value as @@ -114,6 +166,8 @@ always available. This function should be used for internal and specialized purposes only. + .. audit-event:: sys._current_frames + .. function:: breakpointhook() @@ -248,16 +302,23 @@ always available. before the program exits. The handling of such top-level exceptions can be customized by assigning another three-argument function to ``sys.excepthook``. + .. seealso:: + + The :func:`sys.unraisablehook` function handles unraisable exceptions + and the :func:`threading.excepthook` function handles exception raised + by :func:`threading.Thread.run`. + .. data:: __breakpointhook__ __displayhook__ __excepthook__ + __unraisablehook__ These objects contain the original values of ``breakpointhook``, - ``displayhook``, and ``excepthook`` at the start of the program. They are - saved so that ``breakpointhook``, ``displayhook`` and ``excepthook`` can be - restored in case they happen to get replaced with broken or alternative - objects. + ``displayhook``, ``excepthook``, and ``unraisablehook`` at the start of the + program. They are saved so that ``breakpointhook``, ``displayhook`` and + ``excepthook``, ``unraisablehook`` can be restored in case they happen to + get replaced with broken or alternative objects. .. versionadded:: 3.7 __breakpointhook__ @@ -614,6 +675,8 @@ always available. that is deeper than the call stack, :exc:`ValueError` is raised. The default for *depth* is zero, returning the frame at the top of the call stack. + .. audit-event:: sys._getframe + .. impl-detail:: This function should be used for internal and specialized purposes only. @@ -722,22 +785,6 @@ always available. for details.) Use it only for debugging purposes. -.. function:: get_coroutine_wrapper() - - Returns ``None``, or a wrapper set by :func:`set_coroutine_wrapper`. - - .. versionadded:: 3.5 - See :pep:`492` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - - .. deprecated:: 3.7 - The coroutine wrapper functionality has been deprecated, and - will be removed in 3.8. See :issue:`32591` for details. - - .. data:: hash_info A :term:`struct sequence` giving parameters of the numeric hash @@ -1143,6 +1190,8 @@ always available. ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends on the event type. + .. audit-event:: sys.setprofile + The events have the following meaning: ``'call'`` @@ -1263,6 +1312,8 @@ always available. For more information on code and frame objects, refer to :ref:`types`. + .. audit-event:: sys.settrace + .. impl-detail:: The :func:`settrace` function is intended only for implementing debuggers, @@ -1283,6 +1334,13 @@ always available. first time. The *finalizer* will be called when an asynchronous generator is about to be garbage collected. + .. audit-event:: sys.set_asyncgen_hooks_firstiter + + .. audit-event:: sys.set_asyncgen_hooks_finalizer + + Two auditing events are raised because the underlying API consists of two + calls, each of which must raise its own event. + .. versionadded:: 3.6 See :pep:`525` for more details, and for a reference example of a *finalizer* method see the implementation of @@ -1314,49 +1372,6 @@ always available. This function has been added on a provisional basis (see :pep:`411` for details.) Use it only for debugging purposes. -.. function:: set_coroutine_wrapper(wrapper) - - Allows intercepting creation of :term:`coroutine` objects (only ones that - are created by an :keyword:`async def` function; generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine` will not be - intercepted). - - The *wrapper* argument must be either: - - * a callable that accepts one argument (a coroutine object); - * ``None``, to reset the wrapper. - - If called twice, the new wrapper replaces the previous one. The function - is thread-specific. - - The *wrapper* callable cannot define new coroutines directly or indirectly:: - - def wrapper(coro): - async def wrap(coro): - return await coro - return wrap(coro) - sys.set_coroutine_wrapper(wrapper) - - async def foo(): - pass - - # The following line will fail with a RuntimeError, because - # ``wrapper`` creates a ``wrap(coro)`` coroutine: - foo() - - See also :func:`get_coroutine_wrapper`. - - .. versionadded:: 3.5 - See :pep:`492` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - - .. deprecated:: 3.7 - The coroutine wrapper functionality has been deprecated, and - will be removed in 3.8. See :issue:`32591` for details. - .. function:: _enablelegacywindowsfsencoding() Changes the default filesystem encoding and errors mode to 'mbcs' and @@ -1487,6 +1502,41 @@ always available. is suppressed and only the exception type and value are printed. +.. function:: unraisablehook(unraisable, /) + + Handle an unraisable exception. + + Called when an exception has occurred but there is no way for Python to + handle it. For example, when a destructor raises an exception or during + garbage collection (:func:`gc.collect`). + + The *unraisable* argument has the following attributes: + + * *exc_type*: Exception type. + * *exc_value*: Exception value, can be ``None``. + * *exc_traceback*: Exception traceback, can be ``None``. + * *err_msg*: Error message, can be ``None``. + * *object*: Object causing the exception, can be ``None``. + + The default hook formats *err_msg* and *object* as: + ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message + if *err_msg* is ``None``. + + :func:`sys.unraisablehook` can be overridden to control how unraisable + exceptions are handled. + + Storing *exc_value* using a custom hook can create a reference cycle. It + should be cleared explicitly to break the reference cycle when the + exception is no longer needed. + + Storing *object* using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing *object* after the custom + hook completes to avoid resurrecting objects. + + See also :func:`excepthook` which handles uncaught exceptions. + + .. versionadded:: 3.8 + .. data:: version A string containing the version number of the Python interpreter plus additional diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 054521dcc5d3eb..920c018084b81c 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1081,6 +1081,34 @@ The :mod:`test.support` module defines the following functions: :exc:`PermissionError` is raised. +.. function:: catch_unraisable_exception() + + Context manager catching unraisable exception using + :func:`sys.unraisablehook`. + + Storing the exception value (``cm.unraisable.exc_value``) creates a + reference cycle. The reference cycle is broken explicitly when the context + manager exits. + + Storing the object (``cm.unraisable.object``) can resurrect it if it is set + to an object which is being finalized. Exiting the context manager clears + the stored object. + + Usage:: + + with support.catch_unraisable_exception() as cm: + # code creating an "unraisable exception" + ... + + # check the unraisable exception: use cm.unraisable + ... + + # cm.unraisable attribute no longer exists at this point + # (to break a reference cycle) + + .. versionadded:: 3.8 + + .. function:: find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM) Returns an unused port that should be suitable for binding. This is diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 715940c1c52bfc..f80eb22e18fca4 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -38,6 +38,40 @@ This module defines the following functions: returned. +.. function:: excepthook(args, /) + + Handle uncaught exception raised by :func:`Thread.run`. + + The *args* argument has the following attributes: + + * *exc_type*: Exception type. + * *exc_value*: Exception value, can be ``None``. + * *exc_traceback*: Exception traceback, can be ``None``. + * *thread*: Thread which raised the exception, can be ``None``. + + If *exc_type* is :exc:`SystemExit`, the exception is silently ignored. + Otherwise, the exception is printed out on :data:`sys.stderr`. + + If this function raises an exception, :func:`sys.excepthook` is called to + handle it. + + :func:`threading.excepthook` can be overridden to control how uncaught + exceptions raised by :func:`Thread.run` are handled. + + Storing *exc_value* using a custom hook can create a reference cycle. It + should be cleared explicitly to break the reference cycle when the + exception is no longer needed. + + Storing *object* using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing *object* after the custom + hook completes to avoid resurrecting objects. + + .. seealso:: + :func:`sys.excepthook` handles uncaught exceptions. + + .. versionadded:: 3.8 + + .. function:: get_ident() Return the 'thread identifier' of the current thread. This is a nonzero @@ -56,7 +90,7 @@ This module defines the following functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. .. versionadded:: 3.8 @@ -191,6 +225,10 @@ called is terminated. A thread has a name. The name can be passed to the constructor, and read or changed through the :attr:`~Thread.name` attribute. +If the :meth:`~Thread.run` method raises an exception, +:func:`threading.excepthook` is called to handle it. By default, +:func:`threading.excepthook` ignores silently :exc:`SystemExit`. + A thread can be flagged as a "daemon thread". The significance of this flag is that the entire Python program exits when only daemon threads are left. The initial value is inherited from the creating thread. The flag can be set @@ -242,6 +280,8 @@ since it is impossible to detect the termination of alien threads. base class constructor (``Thread.__init__()``) before doing anything else to the thread. + Daemon threads must not be used in subinterpreters. + .. versionchanged:: 3.3 Added the *daemon* argument. @@ -256,6 +296,12 @@ since it is impossible to detect the termination of alien threads. This method will raise a :exc:`RuntimeError` if called more than once on the same thread object. + Raise a :exc:`RuntimeError` if the thread is a daemon thread and the + method is called from a subinterpreter. + + .. versionchanged:: 3.9 + In a subinterpreter, spawning a daemon thread now raises an exception. + .. method:: run() Method representing the thread's activity. @@ -311,12 +357,13 @@ since it is impossible to detect the termination of alien threads. .. attribute:: native_id - The native integral thread ID of this thread or ``0`` if the thread has not - been started. This is a non-negative integer. See the - :func:`get_native_id` function. + The native integral thread ID of this thread. + This is a non-negative integer, or ``None`` if the thread has not + been started. See the :func:`get_native_id` function. This represents the Thread ID (``TID``) as assigned to the thread by the OS (kernel). Its value may be used to uniquely identify - this particular thread system-wide. + this particular thread system-wide (until the thread terminates, + after which the value may be recycled by the OS). .. note:: @@ -324,7 +371,7 @@ since it is impossible to detect the termination of alien threads. system-wide) from the time the thread is created until the thread has been terminated. - .. availability:: Windows, FreeBSD, Linux, macOS. + .. availability:: Require :func:`get_native_id` function. .. versionadded:: 3.8 diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 60cf892e0888b7..640fb2ec61d359 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -55,7 +55,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. `Tcl/Tk recent man pages `_ Recent Tcl/Tk manuals on www.tcl.tk. - `ActiveState Tcl Home Page `_ + `ActiveState Tcl Home Page `_ The Tk/Tcl development is largely taking place at ActiveState. `Tcl and the Tk Toolkit `_ diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 111289c767f35c..b208ba46d17d99 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -39,8 +39,8 @@ The primary entry point is a :term:`generator`: column where the token begins in the source; a 2-tuple ``(erow, ecol)`` of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed (the last tuple item) - is the *logical* line; continuation lines are included. The 5 tuple is - returned as a :term:`named tuple` with the field names: + is the *physical* line. The 5 tuple is returned as a :term:`named tuple` + with the field names: ``type string start end line``. The returned :term:`named tuple` has an additional property named diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index 5cb7029adf5e9e..c2732d900bc138 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -42,6 +42,9 @@ all Python modules imported during the execution into the current directory. Display the version of the module and exit. +.. versionadded:: 3.8 + Added ``--module`` option that allows to run an executable module. + Main options ^^^^^^^^^^^^ @@ -163,7 +166,7 @@ Programmatic Interface environments. If not defined, *globals* and *locals* default to empty dictionaries. - .. method:: runfunc(func, *args, **kwds) + .. method:: runfunc(func, /, *args, **kwds) Call *func* with the given arguments under control of the :class:`Trace` object with the current tracing parameters. diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 3d90d3cbd974f5..7f9f0c34386799 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -20,8 +20,8 @@ Introduction ============ Turtle graphics is a popular way for introducing programming to kids. It was -part of the original Logo programming language developed by Wally Feurzig and -Seymour Papert in 1966. +part of the original Logo programming language developed by Wally Feurzeig, +Seymour Papert and Cynthia Solomon in 1967. Imagine a robotic turtle starting at (0, 0) in the x-y plane. After an ``import turtle``, give it the command ``turtle.forward(15)``, and it moves (on-screen!) 15 pixels in the diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 07c3a2e7f68276..a21fb44dda5dec 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -98,6 +98,9 @@ the types that arise only incidentally during processing such as the Typical use of these names is for :func:`isinstance` or :func:`issubclass` checks. + +If you instantiate any of these types, note that signatures may vary between Python versions. + Standard names are defined for the following types: .. data:: FunctionType @@ -327,7 +330,7 @@ Additional Utility Classes and Functions The type is roughly equivalent to the following code:: class SimpleNamespace: - def __init__(self, **kwargs): + def __init__(self, /, **kwargs): self.__dict__.update(kwargs) def __repr__(self): diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c2523ed5296003..1a766c29a57a53 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -17,7 +17,8 @@ -------------- -This module supports type hints as specified by :pep:`484` and :pep:`526`. +This module provides runtime support for type hints as specified by +:pep:`484`, :pep:`526`, :pep:`544`, :pep:`586`, :pep:`589`, and :pep:`591`. The most fundamental support consists of the types :data:`Any`, :data:`Union`, :data:`Tuple`, :data:`Callable`, :class:`TypeVar`, and :class:`Generic`. For full specification please see :pep:`484`. For @@ -392,6 +393,48 @@ it as a return value) of a more specialized type is a type error. For example:: Use :class:`object` to indicate that a value could be any type in a typesafe manner. Use :data:`Any` to indicate that a value is dynamically typed. + +Nominal vs structural subtyping +------------------------------- + +Initially :pep:`484` defined Python static type system as using +*nominal subtyping*. This means that a class ``A`` is allowed where +a class ``B`` is expected if and only if ``A`` is a subclass of ``B``. + +This requirement previously also applied to abstract base classes, such as +:class:`Iterable`. The problem with this approach is that a class had +to be explicitly marked to support them, which is unpythonic and unlike +what one would normally do in idiomatic dynamically typed Python code. +For example, this conforms to the :pep:`484`:: + + from typing import Sized, Iterable, Iterator + + class Bucket(Sized, Iterable[int]): + ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... + +:pep:`544` allows to solve this problem by allowing users to write +the above code without explicit base classes in the class definition, +allowing ``Bucket`` to be implicitly considered a subtype of both ``Sized`` +and ``Iterable[int]`` by static type checkers. This is known as +*structural subtyping* (or static duck-typing):: + + from typing import Iterator, Iterable + + class Bucket: # Note: no base classes + ... + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... + + def collect(items: Iterable[int]) -> int: ... + result = collect(Bucket()) # Passes type check + +Moreover, by subclassing a special class :class:`Protocol`, a user +can define new custom protocols to fully enjoy structural subtyping +(see examples below). + + Classes, functions, and decorators ---------------------------------- @@ -459,6 +502,39 @@ The module defines the following classes, functions and decorators: except KeyError: return default +.. class:: Protocol(Generic) + + Base class for protocol classes. Protocol classes are defined like this:: + + class Proto(Protocol): + def meth(self) -> int: + ... + + Such classes are primarily used with static type checkers that recognize + structural subtyping (static duck-typing), for example:: + + class C: + def meth(self) -> int: + return 0 + + def func(x: Proto) -> int: + return x.meth() + + func(C()) # Passes static type check + + See :pep:`544` for details. Protocol classes decorated with + :func:`runtime_checkable` (described later) act as simple-minded runtime + protocols that check only the presence of given attributes, ignoring their + type signatures. + + Protocol classes can be generic, for example:: + + class GenProto(Protocol[T]): + def meth(self) -> T: + ... + + .. versionadded:: 3.8 + .. class:: Type(Generic[CT_co]) A variable annotated with ``C`` may accept a value of type ``C``. In @@ -529,6 +605,12 @@ The module defines the following classes, functions and decorators: An ABC with one abstract method ``__bytes__``. +.. class:: SupportsIndex + + An ABC with one abstract method ``__index__``. + + .. versionadded:: 3.8 + .. class:: SupportsAbs An ABC with one abstract method ``__abs__`` that is covariant @@ -555,7 +637,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.Collection` - .. versionadded:: 3.6 + .. versionadded:: 3.6.0 .. class:: AbstractSet(Sized, Collection[T_co]) @@ -599,6 +681,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.deque`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: List(list, MutableSequence[T]) @@ -648,6 +731,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.Awaitable`. + .. versionadded:: 3.5.2 + .. class:: Coroutine(Awaitable[V_co], Generic[T_co T_contra, V_co]) A generic version of :class:`collections.abc.Coroutine`. @@ -661,25 +746,33 @@ The module defines the following classes, functions and decorators: async def bar() -> None: x = await c # type: int + .. versionadded:: 3.5.3 + .. class:: AsyncIterable(Generic[T_co]) A generic version of :class:`collections.abc.AsyncIterable`. + .. versionadded:: 3.5.2 + .. class:: AsyncIterator(AsyncIterable[T_co]) A generic version of :class:`collections.abc.AsyncIterator`. + .. versionadded:: 3.5.2 + .. class:: ContextManager(Generic[T_co]) A generic version of :class:`contextlib.AbstractContextManager`. - .. versionadded:: 3.6 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.0 .. class:: AsyncContextManager(Generic[T_co]) A generic version of :class:`contextlib.AbstractAsyncContextManager`. - .. versionadded:: 3.6 + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 .. class:: Dict(dict, MutableMapping[KT, VT]) @@ -708,12 +801,14 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.Counter`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) A generic version of :class:`collections.ChainMap`. + .. versionadded:: 3.5.4 .. versionadded:: 3.6.1 .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -778,7 +873,7 @@ The module defines the following classes, functions and decorators: yield start start = await increment(start) - .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 .. class:: Text @@ -872,6 +967,39 @@ The module defines the following classes, functions and decorators: The ``_field_types`` and ``__annotations__`` attributes are now regular dictionaries instead of instances of ``OrderedDict``. +.. class:: TypedDict(dict) + + A simple typed namespace. At runtime it is equivalent to + a plain :class:`dict`. + + ``TypedDict`` creates a dictionary type that expects all of its + instances to have a certain set of keys, where each key is + associated with a value of a consistent type. This expectation + is not checked at runtime but is only enforced by type checkers. + Usage:: + + class Point2D(TypedDict): + x: int + y: int + label: str + + a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK + b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + + assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + + The type info for introspection can be accessed via ``Point2D.__annotations__`` + and ``Point2D.__total__``. To allow using this feature with older versions + of Python that do not support :pep:`526`, ``TypedDict`` supports two additional + equivalent syntactic forms:: + + Point2D = TypedDict('Point2D', x=int, y=int, label=str) + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) + + See :pep:`589` for more examples and detailed rules of using ``TypedDict`` + with type checkers. + + .. versionadded:: 3.8 .. function:: NewType(typ) @@ -906,6 +1034,25 @@ The module defines the following classes, functions and decorators: a dictionary constructed by merging all the ``__annotations__`` along ``C.__mro__`` in reverse order. +.. function:: get_origin(typ) +.. function:: get_args(typ) + + Provide basic introspection for generic types and special typing forms. + + For a typing object of the form ``X[Y, Z, ...]`` these functions return + ``X`` and ``(Y, Z, ...)``. If ``X`` is a generic alias for a builtin or + :mod:`collections` class, it gets normalized to the original class. + For unsupported objects return ``None`` and ``()`` correspondingly. + Examples:: + + assert get_origin(Dict[str, int]) is dict + assert get_args(Dict[int, str]) == (int, str) + + assert get_origin(Union[int, str]) is Union + assert get_args(Union[int, str]) == (int, str) + + .. versionadded:: 3.8 + .. decorator:: overload The ``@overload`` decorator allows describing functions and methods @@ -934,6 +1081,31 @@ The module defines the following classes, functions and decorators: See :pep:`484` for details and comparison with other typing semantics. +.. decorator:: final + + A decorator to indicate to type checkers that the decorated method + cannot be overridden, and the decorated class cannot be subclassed. + For example:: + + class Base: + @final + def done(self) -> None: + ... + class Sub(Base): + def done(self) -> None: # Error reported by type checker + ... + + @final + class Leaf: + ... + class Other(Leaf): # Error reported by type checker + ... + + There is no runtime checking of these properties. See :pep:`591` for + more details. + + .. versionadded:: 3.8 + .. decorator:: no_type_check Decorator to indicate that annotations are not type hints. @@ -969,6 +1141,26 @@ The module defines the following classes, functions and decorators: Note that returning instances of private classes is not recommended. It is usually preferable to make such classes public. +.. decorator:: runtime_checkable + + Mark a protocol class as a runtime protocol. + + Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. + This raises :exc:`TypeError` when applied to a non-protocol class. This + allows a simple-minded structural check, very similar to "one trick ponies" + in :mod:`collections.abc` such as :class:`Iterable`. For example:: + + @runtime_checkable + class Closable(Protocol): + def close(self): ... + + assert isinstance(open('/some/file'), Closable) + + **Warning:** this will check only the presence of the required methods, + not their type signatures! + + .. versionadded:: 3.8 + .. data:: Any Special type indicating an unconstrained type. @@ -987,6 +1179,7 @@ The module defines the following classes, functions and decorators: raise RuntimeError('no way') .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 .. data:: Union @@ -1072,6 +1265,28 @@ The module defines the following classes, functions and decorators: ``Callable[..., Any]``, and in turn to :class:`collections.abc.Callable`. +.. data:: Literal + + A type that can be used to indicate to type checkers that the + corresponding variable or function parameter has a value equivalent to + the provided literal (or one of several literals). For example:: + + def validate_simple(data: Any) -> Literal[True]: # always returns True + ... + + MODE = Literal['r', 'rb', 'w', 'wb'] + def open_helper(file: str, mode: MODE) -> str: + ... + + open_helper('/some/path', 'r') # Passes type check + open_helper('/other/path', 'typo') # Error in type checker + + ``Literal[...]`` cannot be subclassed. At runtime, an arbitrary value + is allowed as type argument to ``Literal[...]``, but type checkers may + impose restrictions. See :pep:`586` for more details about literal types. + + .. versionadded:: 3.8 + .. data:: ClassVar Special type construct to mark class variables. @@ -1098,6 +1313,25 @@ The module defines the following classes, functions and decorators: .. versionadded:: 3.5.3 +.. data:: Final + + A special typing construct to indicate to type checkers that a name + cannot be re-assigned or overridden in a subclass. For example:: + + MAX_SIZE: Final = 9000 + MAX_SIZE += 1 # Error reported by type checker + + class Connection: + TIMEOUT: Final[int] = 10 + + class FastConnector(Connection): + TIMEOUT = 1 # Error reported by type checker + + There is no runtime checking of these properties. See :pep:`591` for + more details. + + .. versionadded:: 3.8 + .. data:: AnyStr ``AnyStr`` is a type variable defined as diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 16690f349822ec..811f0fb1ce9397 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -848,7 +848,7 @@ Here's an example implementation: >>> from copy import deepcopy >>> class CopyingMock(MagicMock): - ... def __call__(self, *args, **kwargs): + ... def __call__(self, /, *args, **kwargs): ... args = deepcopy(args) ... kwargs = deepcopy(kwargs) ... return super(CopyingMock, self).__call__(*args, **kwargs) @@ -1042,7 +1042,7 @@ that it takes arbitrary keyword arguments (``**kwargs``) which are then passed onto the mock constructor: >>> class Subclass(MagicMock): - ... def _get_child_mock(self, **kwargs): + ... def _get_child_mock(self, /, **kwargs): ... return MagicMock(**kwargs) ... >>> mymock = Subclass() diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index ed00ee6d0c2d83..46e8ef38ab11bd 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -201,9 +201,11 @@ The Mock Class .. testsetup:: + import asyncio + import inspect import unittest from unittest.mock import sentinel, DEFAULT, ANY - from unittest.mock import patch, call, Mock, MagicMock, PropertyMock + from unittest.mock import patch, call, Mock, MagicMock, PropertyMock, AsyncMock from unittest.mock import mock_open :class:`Mock` is a flexible mock object intended to replace the use of stubs and @@ -851,6 +853,217 @@ object:: >>> p.assert_called_once_with() +.. class:: AsyncMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs) + + An asynchronous version of :class:`Mock`. The :class:`AsyncMock` object will + behave so the object is recognized as an async function, and the result of a + call is an awaitable. + + >>> mock = AsyncMock() + >>> asyncio.iscoroutinefunction(mock) + True + >>> inspect.isawaitable(mock()) # doctest: +SKIP + True + + The result of ``mock()`` is an async function which will have the outcome + of ``side_effect`` or ``return_value``: + + - if ``side_effect`` is a function, the async function will return the + result of that function, + - if ``side_effect`` is an exception, the async function will raise the + exception, + - if ``side_effect`` is an iterable, the async function will return the + next value of the iterable, however, if the sequence of result is + exhausted, ``StopIteration`` is raised immediately, + - if ``side_effect`` is not defined, the async function will return the + value defined by ``return_value``, hence, by default, the async function + returns a new :class:`AsyncMock` object. + + + Setting the *spec* of a :class:`Mock` or :class:`MagicMock` to an async function + will result in a coroutine object being returned after calling. + + >>> async def async_func(): pass + ... + >>> mock = MagicMock(async_func) + >>> mock + + >>> mock() # doctest: +SKIP + + + .. method:: assert_awaited() + + Assert that the mock was awaited at least once. + + >>> mock = AsyncMock() + >>> async def main(): + ... await mock() + ... + >>> asyncio.run(main()) + >>> mock.assert_awaited() + >>> mock_2 = AsyncMock() + >>> mock_2.assert_awaited() + Traceback (most recent call last): + ... + AssertionError: Expected mock to have been awaited. + + .. method:: assert_awaited_once() + + Assert that the mock was awaited exactly once. + + >>> mock = AsyncMock() + >>> async def main(): + ... await mock() + ... + >>> asyncio.run(main()) + >>> mock.assert_awaited_once() + >>> asyncio.run(main()) + >>> mock.method.assert_awaited_once() + Traceback (most recent call last): + ... + AssertionError: Expected mock to have been awaited once. Awaited 2 times. + + .. method:: assert_awaited_with(*args, **kwargs) + + Assert that the last await was with the specified arguments. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> asyncio.run(main('foo', bar='bar')) + >>> mock.assert_awaited_with('foo', bar='bar') + >>> mock.assert_awaited_with('other') + Traceback (most recent call last): + ... + AssertionError: expected call not found. + Expected: mock('other') + Actual: mock('foo', bar='bar') + + .. method:: assert_awaited_once_with(*args, **kwargs) + + Assert that the mock was awaited exactly once and with the specified + arguments. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> asyncio.run(main('foo', bar='bar')) + >>> mock.assert_awaited_once_with('foo', bar='bar') + >>> asyncio.run(main('foo', bar='bar')) + >>> mock.assert_awaited_once_with('foo', bar='bar') + Traceback (most recent call last): + ... + AssertionError: Expected mock to have been awaited once. Awaited 2 times. + + .. method:: assert_any_await(*args, **kwargs) + + Assert the mock has ever been awaited with the specified arguments. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> asyncio.run(main('foo', bar='bar')) + >>> asyncio.run(main('hello')) + >>> mock.assert_any_await('foo', bar='bar') + >>> mock.assert_any_await('other') + Traceback (most recent call last): + ... + AssertionError: mock('other') await not found + + .. method:: assert_has_awaits(calls, any_order=False) + + Assert the mock has been awaited with the specified calls. + The :attr:`await_args_list` list is checked for the awaits. + + If *any_order* is False (the default) then the awaits must be + sequential. There can be extra calls before or after the + specified awaits. + + If *any_order* is True then the awaits can be in any order, but + they must all appear in :attr:`await_args_list`. + + >>> mock = AsyncMock() + >>> async def main(*args, **kwargs): + ... await mock(*args, **kwargs) + ... + >>> calls = [call("foo"), call("bar")] + >>> mock.assert_has_calls(calls) + Traceback (most recent call last): + ... + AssertionError: Calls not found. + Expected: [call('foo'), call('bar')] + >>> asyncio.run(main('foo')) + >>> asyncio.run(main('bar')) + >>> mock.assert_has_calls(calls) + + .. method:: assert_not_awaited() + + Assert that the mock was never awaited. + + >>> mock = AsyncMock() + >>> mock.assert_not_awaited() + + .. method:: reset_mock(*args, **kwargs) + + See :func:`Mock.reset_mock`. Also sets :attr:`await_count` to 0, + :attr:`await_args` to None, and clears the :attr:`await_args_list`. + + .. attribute:: await_count + + An integer keeping track of how many times the mock object has been awaited. + + >>> mock = AsyncMock() + >>> async def main(): + ... await mock() + ... + >>> asyncio.run(main()) + >>> mock.await_count + 1 + >>> asyncio.run(main()) + >>> mock.await_count + 2 + + .. attribute:: await_args + + This is either ``None`` (if the mock hasn’t been awaited), or the arguments that + the mock was last awaited with. Functions the same as :attr:`Mock.call_args`. + + >>> mock = AsyncMock() + >>> async def main(*args): + ... await mock(*args) + ... + >>> mock.await_args + >>> asyncio.run(main('foo')) + >>> mock.await_args + call('foo') + >>> asyncio.run(main('bar')) + >>> mock.await_args + call('bar') + + + .. attribute:: await_args_list + + This is a list of all the awaits made to the mock object in sequence (so the + length of the list is the number of times it has been awaited). Before any + awaits have been made it is an empty list. + + >>> mock = AsyncMock() + >>> async def main(*args): + ... await mock(*args) + ... + >>> mock.await_args_list + [] + >>> asyncio.run(main('foo')) + >>> mock.await_args_list + [call('foo')] + >>> asyncio.run(main('bar')) + >>> mock.await_args_list + [call('foo'), call('bar')] + + Calling ~~~~~~~ @@ -1343,15 +1556,24 @@ patch.dict decorator. When used as a class decorator :func:`patch.dict` honours ``patch.TEST_PREFIX`` for choosing which methods to wrap. + .. versionchanged:: 3.8 + + :func:`patch.dict` now returns the patched dictionary when used as a context + manager. + :func:`patch.dict` can be used to add members to a dictionary, or simply let a test change a dictionary, and ensure the dictionary is restored when the test ends. >>> foo = {} - >>> with patch.dict(foo, {'newkey': 'newvalue'}): + >>> with patch.dict(foo, {'newkey': 'newvalue'}) as patched_foo: ... assert foo == {'newkey': 'newvalue'} + ... assert patched_foo == {'newkey': 'newvalue'} + ... # You can add, update or delete keys of foo (or patched_foo, it's the same dict) + ... patched_foo['spam'] = 'eggs' ... >>> assert foo == {} + >>> assert patched_foo == {} >>> import os >>> with patch.dict('os.environ', {'newkey': 'newvalue'}): @@ -1732,7 +1954,7 @@ The full list of supported magic methods is: * Container methods: ``__getitem__``, ``__setitem__``, ``__delitem__``, ``__contains__``, ``__len__``, ``__iter__``, ``__reversed__`` and ``__missing__`` -* Context manager: ``__enter__`` and ``__exit__`` +* Context manager: ``__enter__``, ``__exit__``, ``__aenter__`` and ``__aexit__`` * Unary numeric methods: ``__neg__``, ``__pos__`` and ``__invert__`` * The numeric methods (including right hand and in-place variants): ``__add__``, ``__sub__``, ``__mul__``, ``__matmul__``, ``__div__``, ``__truediv__``, @@ -1744,10 +1966,14 @@ The full list of supported magic methods is: * Pickling: ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, ``__getnewargs__``, ``__getstate__`` and ``__setstate__`` * File system path representation: ``__fspath__`` +* Asynchronous iteration methods: ``__aiter__`` and ``__anext__`` .. versionchanged:: 3.8 Added support for :func:`os.PathLike.__fspath__`. +.. versionchanged:: 3.8 + Added support for ``__aenter__``, ``__aexit__``, ``__aiter__`` and ``__anext__``. + The following methods exist but are *not* supported as they are either in use by mock, can't be set dynamically, or can cause problems: @@ -1810,6 +2036,7 @@ Methods and their defaults: * ``__len__``: 0 * ``__iter__``: iter([]) * ``__exit__``: False +* ``__aexit__``: False * ``__complex__``: 1j * ``__float__``: 1.0 * ``__bool__``: True diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 8ad2abd3d89a44..320d898fc8d4ca 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -510,7 +510,8 @@ that is broken and will fail, but shouldn't be counted as a failure on a :class:`TestResult`. Skipping a test is simply a matter of using the :func:`skip` :term:`decorator` -or one of its conditional variants. +or one of its conditional variants, calling :meth:`TestCase.skipTest` within a +:meth:`~TestCase.setUp` or test method, or raising :exc:`SkipTest` directly. Basic skipping looks like this:: @@ -531,16 +532,23 @@ Basic skipping looks like this:: # windows specific testing code pass + def test_maybe_skipped(self): + if not external_resource_available(): + self.skipTest("external resource not available") + # test code that depends on the external resource + pass + This is the output of running the example above in verbose mode:: test_format (__main__.MyTestCase) ... skipped 'not supported in this library version' test_nothing (__main__.MyTestCase) ... skipped 'demonstrating skipping' + test_maybe_skipped (__main__.MyTestCase) ... skipped 'external resource not available' test_windows_support (__main__.MyTestCase) ... skipped 'requires Windows' ---------------------------------------------------------------------- - Ran 3 tests in 0.005s + Ran 4 tests in 0.005s - OK (skipped=3) + OK (skipped=4) Classes can be skipped just like methods:: @@ -568,7 +576,7 @@ the test unless the passed object has a certain attribute:: return lambda func: func return unittest.skip("{!r} doesn't have {!r}".format(obj, attr)) -The following decorators implement test skipping and expected failures: +The following decorators and exception implement test skipping and expected failures: .. decorator:: skip(reason) @@ -1419,7 +1427,7 @@ Test cases :class:`TextTestResult` in Python 3.2. - .. method:: addCleanup(function, *args, **kwargs) + .. method:: addCleanup(function, /, *args, **kwargs) Add a function to be called after :meth:`tearDown` to cleanup resources used during the test. Functions will be called in reverse order to the @@ -1448,7 +1456,7 @@ Test cases .. versionadded:: 3.1 - .. classmethod:: addClassCleanup(function, *args, **kwargs) + .. classmethod:: addClassCleanup(function, /, *args, **kwargs) Add a function to be called after :meth:`tearDownClass` to cleanup resources used during the test class. Functions will be called in reverse @@ -2305,7 +2313,7 @@ To add cleanup code that must be run even in the case of an exception, use ``addModuleCleanup``: -.. function:: addModuleCleanup(function, *args, **kwargs) +.. function:: addModuleCleanup(function, /, *args, **kwargs) Add a function to be called after :func:`tearDownModule` to cleanup resources used during the test class. Functions will be called in reverse diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index f9936288fd42cd..49276daa7ff43f 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -370,6 +370,13 @@ or on combining URL components into a URL string. .. versionchanged:: 3.2 Result is a structured object rather than a simple 2-tuple. +.. function:: unwrap(url) + + Extract the url from a wrapped URL (that is, a string formatted as + ````, ````, ``URL:scheme://host/path`` + or ``scheme://host/path``). If *url* is not a wrapped URL, it is returned + without changes. + .. _parsing-ascii-encoded-bytes: Parsing ASCII Encoded Bytes diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 14fa27bb08af20..a53c969ec629a5 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -95,6 +95,12 @@ The :mod:`urllib.request` module defines the following functions: parameter to ``urllib.urlopen``, can be obtained by using :class:`ProxyHandler` objects. + .. audit-event:: urllib.Request "fullurl data headers method" + + The default opener raises an :func:`auditing event ` + ``urllib.Request`` with arguments ``fullurl``, ``data``, ``headers``, + ``method`` taken from the request object. + .. versionchanged:: 3.2 *cafile* and *capath* were added. @@ -118,6 +124,7 @@ The :mod:`urllib.request` module defines the following functions: :func:`ssl.create_default_context` select the system's trusted CA certificates for you. + .. function:: install_opener(opener) Install an :class:`OpenerDirector` instance as the default global opener. diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index d121f320d6a3eb..a481a3509d4ec8 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -19,10 +19,10 @@ Python programmers issue warnings by calling the :func:`warn` function defined in this module. (C programmers use :c:func:`PyErr_WarnEx`; see :ref:`exceptionhandling` for details). -Warning messages are normally written to ``sys.stderr``, but their disposition +Warning messages are normally written to :data:`sys.stderr`, but their disposition can be changed flexibly, from ignoring all warnings to turning them into -exceptions. The disposition of warnings can vary based on the warning category -(see below), the text of the warning message, and the source location where it +exceptions. The disposition of warnings can vary based on the :ref:`warning category +`, the text of the warning message, and the source location where it is issued. Repetitions of a particular warning for the same source location are typically suppressed. @@ -31,7 +31,7 @@ determination is made whether a message should be issued or not; next, if a message is to be issued, it is formatted and printed using a user-settable hook. The determination whether to issue a warning message is controlled by the -warning filter, which is a sequence of matching rules and actions. Rules can be +:ref:`warning filter `, which is a sequence of matching rules and actions. Rules can be added to the filter by calling :func:`filterwarnings` and reset to its default state by calling :func:`resetwarnings`. @@ -181,9 +181,9 @@ Describing Warning Filters The warnings filter is initialized by :option:`-W` options passed to the Python interpreter command line and the :envvar:`PYTHONWARNINGS` environment variable. The interpreter saves the arguments for all supplied entries without -interpretation in ``sys.warnoptions``; the :mod:`warnings` module parses these +interpretation in :data:`sys.warnoptions`; the :mod:`warnings` module parses these when it is first imported (invalid options are ignored, after printing a -message to ``sys.stderr``). +message to :data:`sys.stderr`). Individual warnings filters are specified as a sequence of fields separated by colons:: @@ -192,7 +192,7 @@ colons:: The meaning of each of these fields is as described in :ref:`warning-filter`. When listing multiple filters on a single line (as for -:envvar:`PYTHONWARNINGS`), the individual filters are separated by commas,and +:envvar:`PYTHONWARNINGS`), the individual filters are separated by commas and the filters listed later take precedence over those listed before them (as they're applied left-to-right, and the most recently applied filters take precedence over earlier ones). @@ -395,12 +395,12 @@ Available Functions .. function:: warn(message, category=None, stacklevel=1, source=None) Issue a warning, or maybe ignore it or raise an exception. The *category* - argument, if given, must be a warning category class (see above); it defaults to - :exc:`UserWarning`. Alternatively *message* can be a :exc:`Warning` instance, + argument, if given, must be a :ref:`warning category class `; it + defaults to :exc:`UserWarning`. Alternatively, *message* can be a :exc:`Warning` instance, in which case *category* will be ignored and ``message.__class__`` will be used. - In this case the message text will be ``str(message)``. This function raises an + In this case, the message text will be ``str(message)``. This function raises an exception if the particular warning issued is changed into an error by the - warnings filter see above. The *stacklevel* argument can be used by wrapper + :ref:`warnings filter `. The *stacklevel* argument can be used by wrapper functions written in Python, like this:: def deprecation(message): @@ -444,7 +444,7 @@ Available Functions Write a warning to a file. The default implementation calls ``formatwarning(message, category, filename, lineno, line)`` and writes the - resulting string to *file*, which defaults to ``sys.stderr``. You may replace + resulting string to *file*, which defaults to :data:`sys.stderr`. You may replace this function with any callable by assigning to ``warnings.showwarning``. *line* is a line of source code to be included in the warning message; if *line* is not supplied, :func:`showwarning` will diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 80a908bbd83b0a..c3519e45beb6b1 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -65,8 +65,8 @@ exposed by the :mod:`weakref` module for the benefit of advanced uses. Not all objects can be weakly referenced; those objects which can include class instances, functions written in Python (but not in C), instance methods, sets, -frozensets, some :term:`file objects `, :term:`generator`\s, type -objects, sockets, arrays, deques, regular expression pattern objects, and code +frozensets, some :term:`file objects `, :term:`generators `, +type objects, sockets, arrays, deques, regular expression pattern objects, and code objects. .. versionchanged:: 3.2 @@ -80,9 +80,10 @@ support weak references but can add support through subclassing:: obj = Dict(red=1, green=2, blue=3) # this object is weak referenceable -Other built-in types such as :class:`tuple` and :class:`int` do not support weak -references even when subclassed (This is an implementation detail and may be -different across various Python implementations.). +.. impl-detail:: + + Other built-in types such as :class:`tuple` and :class:`int` do not support weak + references even when subclassed. Extension types can easily be made to support weak references; see :ref:`weakref-support`. @@ -240,7 +241,7 @@ objects. .. versionadded:: 3.4 -.. class:: finalize(obj, func, *args, **kwargs) +.. class:: finalize(obj, func, /, *args, **kwargs) Return a callable finalizer object which will be called when *obj* is garbage collected. Unlike an ordinary weak reference, a finalizer @@ -396,7 +397,7 @@ the referent is accessed:: import weakref class ExtendedRef(weakref.ref): - def __init__(self, ob, callback=None, **annotations): + def __init__(self, ob, callback=None, /, **annotations): super(ExtendedRef, self).__init__(ob, callback) self.__counter = 0 for k, v in annotations.items(): diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 2423a0c15691f4..8711242d95d741 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -134,11 +134,12 @@ module documentation. This section lists the differences between the API and .. method:: Node.writexml(writer, indent="", addindent="", newl="") - Write XML to the writer object. The writer should have a :meth:`write` method - which matches that of the file object interface. The *indent* parameter is the - indentation of the current node. The *addindent* parameter is the incremental - indentation to use for subnodes of the current one. The *newl* parameter - specifies the string to use to terminate newlines. + Write XML to the writer object. The writer receives texts but not bytes as input, + it should have a :meth:`write` method which matches that of the file object + interface. The *indent* parameter is the indentation of the current node. + The *addindent* parameter is the incremental indentation to use for subnodes + of the current one. The *newl* parameter specifies the string to use to + terminate newlines. For the :class:`Document` node, an additional keyword argument *encoding* can be used to specify the encoding field of the XML header. diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index aa61278e099ac6..339acfd0e5786f 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -47,7 +47,7 @@ The available exception and functions in this module are: platforms, use ``adler32(data) & 0xffffffff``. -.. function:: compress(data, level=-1) +.. function:: compress(data, /, level=-1) Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; @@ -132,7 +132,7 @@ The available exception and functions in this module are: platforms, use ``crc32(data) & 0xffffffff``. -.. function:: decompress(data, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) +.. function:: decompress(data, /, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) Decompresses the bytes in *data*, returning a bytes object containing the uncompressed data. The *wbits* parameter depends on diff --git a/Doc/make.bat b/Doc/make.bat index e6604956ea916b..dfc622f66615df 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -117,13 +117,13 @@ if not exist "%BUILDDIR%" mkdir "%BUILDDIR%" rem PY_MISC_NEWS_DIR is also used by our Sphinx extension in tools/extensions/pyspecific.py if not defined PY_MISC_NEWS_DIR set PY_MISC_NEWS_DIR=%BUILDDIR%\%1 +if not exist "%PY_MISC_NEWS_DIR%" mkdir "%PY_MISC_NEWS_DIR%" if exist ..\Misc\NEWS ( echo.Copying Misc\NEWS to %PY_MISC_NEWS_DIR%\NEWS copy ..\Misc\NEWS "%PY_MISC_NEWS_DIR%\NEWS" > nul ) else if exist ..\Misc\NEWS.D ( if defined BLURB ( echo.Merging Misc/NEWS with %BLURB% - if not exist build mkdir build %BLURB% merge -f "%PY_MISC_NEWS_DIR%\NEWS" ) else ( echo.No Misc/NEWS file and Blurb is not available. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 42fa8647623935..988eec6d254e10 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -483,8 +483,10 @@ A function definition defines a user-defined function object (see section decorators: `decorator`+ decorator: "@" `dotted_name` ["(" [`argument_list` [","]] ")"] NEWLINE dotted_name: `identifier` ("." `identifier`)* - parameter_list: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] - : | `parameter_list_starargs` + parameter_list: `defparameter` ("," `defparameter`)* "," "/" ["," [`parameter_list_no_posonly`]] + : | `parameter_list_no_posonly` + parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] + : | `parameter_list_starargs` parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 9fc9f3a3848a4e..fa47bf1c1619b5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -890,6 +890,8 @@ Internal types .. index:: single: co_argcount (code object attribute) + single: co_posonlyargcount (code object attribute) + single: co_kwonlyargcount (code object attribute) single: co_code (code object attribute) single: co_consts (code object attribute) single: co_filename (code object attribute) @@ -905,21 +907,26 @@ Internal types single: co_freevars (code object attribute) Special read-only attributes: :attr:`co_name` gives the function name; - :attr:`co_argcount` is the number of positional arguments (including arguments - with default values); :attr:`co_nlocals` is the number of local variables used - by the function (including arguments); :attr:`co_varnames` is a tuple containing + :attr:`co_argcount` is the total number of positional arguments + (including positional-only arguments and arguments with default values); + :attr:`co_posonlyargcount` is the number of positional-only arguments + (including arguments with default values); :attr:`co_kwonlyargcount` is + the number of keyword-only arguments (including arguments with default + values); :attr:`co_nlocals` is the number of local variables used by the + function (including arguments); :attr:`co_varnames` is a tuple containing the names of the local variables (starting with the argument names); - :attr:`co_cellvars` is a tuple containing the names of local variables that are - referenced by nested functions; :attr:`co_freevars` is a tuple containing the - names of free variables; :attr:`co_code` is a string representing the sequence - of bytecode instructions; :attr:`co_consts` is a tuple containing the literals - used by the bytecode; :attr:`co_names` is a tuple containing the names used by - the bytecode; :attr:`co_filename` is the filename from which the code was - compiled; :attr:`co_firstlineno` is the first line number of the function; - :attr:`co_lnotab` is a string encoding the mapping from bytecode offsets to - line numbers (for details see the source code of the interpreter); - :attr:`co_stacksize` is the required stack size (including local variables); - :attr:`co_flags` is an integer encoding a number of flags for the interpreter. + :attr:`co_cellvars` is a tuple containing the names of local variables + that are referenced by nested functions; :attr:`co_freevars` is a tuple + containing the names of free variables; :attr:`co_code` is a string + representing the sequence of bytecode instructions; :attr:`co_consts` is + a tuple containing the literals used by the bytecode; :attr:`co_names` is + a tuple containing the names used by the bytecode; :attr:`co_filename` is + the filename from which the code was compiled; :attr:`co_firstlineno` is + the first line number of the function; :attr:`co_lnotab` is a string + encoding the mapping from bytecode offsets to line numbers (for details + see the source code of the interpreter); :attr:`co_stacksize` is the + required stack size (including local variables); :attr:`co_flags` is an + integer encoding a number of flags for the interpreter. .. index:: object: generator @@ -1804,7 +1811,7 @@ class defining the method. class, as in:: class Philosopher: - def __init_subclass__(cls, default_name, **kwargs): + def __init_subclass__(cls, /, default_name, **kwargs): super().__init_subclass__(**kwargs) cls.default_name = default_name @@ -2387,11 +2394,9 @@ left undefined. functions). Presence of this method indicates that the numeric object is an integer type. Must return an integer. - .. note:: - - In order to have a coherent integer type class, when :meth:`__index__` is - defined :meth:`__int__` should also be defined, and both should return - the same value. + If :meth:`__int__`, :meth:`__float__` and :meth:`__complex__` are not + defined then corresponding built-in functions :func:`int`, :func:`float` + and :func:`complex` fall back to :meth:`__index__`. .. method:: object.__round__(self, [,ndigits]) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 52b41929d7bc7c..8b7110615240ee 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1563,14 +1563,15 @@ y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and ``False`` otherwise. For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is -produced while iterating over ``y``. If an exception is raised during the -iteration, it is as if :keyword:`in` raised that exception. +:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z``, for which the +expression ``x is z or x == z`` is true, is produced while iterating over ``y``. +If an exception is raised during the iteration, it is as if :keyword:`in` raised +that exception. Lastly, the old-style iteration protocol is tried: if a class defines :meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative -integer index *i* such that ``x == y[i]``, and all lower integer indices do not -raise :exc:`IndexError` exception. (If any other exception is raised, it is as +integer index *i* such that ``x is y[i] or x == y[i]``, and no lower integer index +raises the :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). .. index:: diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index af7c0caff62797..0a043a90050c4e 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -329,10 +329,10 @@ Annotated assignment statements statement, of a variable or attribute annotation and an optional assignment statement: .. productionlist:: - annotated_assignment_stmt: `augtarget` ":" `expression` ["=" `expression`] + annotated_assignment_stmt: `augtarget` ":" `expression` + : ["=" (`starred_expression` | `yield_expression`)] -The difference from normal :ref:`assignment` is that only single target and -only single right hand side value is allowed. +The difference from normal :ref:`assignment` is that only single target is allowed. For simple names as assignment targets, if in class or module scope, the annotations are evaluated and stored in a special class or module @@ -366,6 +366,11 @@ target, then the interpreter evaluates the target except for the last syntax for type annotations that can be used in static analysis tools and IDEs. +.. versionchanged:: 3.8 + Now annotated assignments allow same expressions in the right hand side as + the regular assignments. Previously, some expressions (like un-parenthesized + tuple expressions) caused a syntax error. + .. _assert: diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index e097c13eab5f4b..f79b25048a5a64 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -151,6 +151,45 @@ def run(self): return [pnode] +# Support for documenting audit event + +class AuditEvent(Directive): + + has_content = True + required_arguments = 1 + optional_arguments = 1 + final_argument_whitespace = True + + _label = [ + "Raises an :ref:`auditing event ` {name} with no arguments.", + "Raises an :ref:`auditing event ` {name} with argument {args}.", + "Raises an :ref:`auditing event ` {name} with arguments {args}.", + ] + + def run(self): + if len(self.arguments) >= 2 and self.arguments[1]: + args = [ + "``{}``".format(a.strip()) + for a in self.arguments[1].strip("'\"").split() + if a.strip() + ] + else: + args = [] + + label = translators['sphinx'].gettext(self._label[min(2, len(args))]) + text = label.format(name="``{}``".format(self.arguments[0]), + args=", ".join(args)) + + pnode = nodes.paragraph(text, classes=["audit-hook"]) + if self.content: + self.state.nested_parse(self.content, self.content_offset, pnode) + else: + n, m = self.state.inline_text(text, self.lineno) + pnode.extend(n + m) + + return [pnode] + + # Support for documenting decorators class PyDecoratorMixin(object): @@ -424,6 +463,7 @@ def setup(app): app.add_role('source', source_role) app.add_directive('impl-detail', ImplementationDetail) app.add_directive('availability', Availability) + app.add_directive('audit-event', AuditEvent) app.add_directive('deprecated-removed', DeprecatedRemoved) app.add_builder(PydocTopicsBuilder) app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index 346b31494e60f9..fa298a76b0fe10 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,7 +10,8 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.8': 'dev (3.8)', + '3.9': 'dev (3.9)', + '3.8': 'pre (3.8)', '3.7': '3.7', '3.6': '3.6', '3.5': '3.5', diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 31b22665eca475..85263d47c8bba8 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -236,6 +236,8 @@ library/urllib.request,,:close,Connection:close library/urllib.request,,:port,:port library/urllib.request,,:lang,"xmlns=""http://www.w3.org/1999/xhtml"" xml:lang=""en"" lang=""en"">\n\n\n" library/urllib.request,,:password,"""joe:password@python.org""" +library/urllib.parse,,:scheme, +library/urllib.parse,,:scheme,URL:scheme://host/path library/uuid,,:uuid,urn:uuid:12345678-1234-5678-1234-567812345678 library/venv,,:param,":param nodist: If True, setuptools and pip are not installed into the" library/venv,,:param,":param progress: If setuptools or pip are installed, the progress of the" @@ -335,7 +337,6 @@ library/zipapp,,:main,"$ python -m zipapp myapp -m ""myapp:main""" library/zipapp,,:fn,"pkg.mod:fn" library/zipapp,,:callable,"pkg.module:callable" library/stdtypes,,::,>>> m[::2].tolist() -library/sys,,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine: whatsnew/3.5,,:root,'WARNING:root:warning\n' whatsnew/3.5,,:warning,'WARNING:root:warning\n' whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') @@ -350,3 +351,5 @@ whatsnew/3.7,,::,error::BytesWarning whatsnew/changelog,,::,error::BytesWarning whatsnew/changelog,,::,default::BytesWarning whatsnew/changelog,,::,default::DeprecationWarning +library/importlib.metadata,,:main,"EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts')" +library/importlib.metadata,,`,of directories ``path`` (defaults to sys.path). diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 3666af92f0da47..4fd7423430ca81 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,7 +2,8 @@

{% trans %}Download{% endtrans %}

{% trans %}Download these documents{% endtrans %}

{% trans %}Docs by version{% endtrans %}